foxhunt/tests/integration/ml_training_service_tests.rs

#![allow(unused_crate_dependencies)]
use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::time::timeout;
use chrono::{DateTime, Utc, TimeZone};

use crate::framework::{TestOrchestrator, TestFrameworkConfig, PerformanceThresholds, IntegrationTestResult};
use crate::framework::mocks::MockServiceRegistry;

use config::{ConfigManager, MLConfig, ModelConfig};
use ml::models::{ModelType, TrainingConfig, ModelMetrics, ModelVersion};
use ml::data::{TrainingDataset, FeatureSet, TargetVariable};

/// ML Training Service Integration Tests
///
/// Tests the ML training service functionality including:
/// - Model training pipeline execution
///
/// - Data preprocessing and feature engineering
/// - Model validation and metrics calculation
///
/// - Model versioning and storage
/// - Distributed training coordination
///
/// - Model deployment and serving
pub struct MLTrainingServiceTests {
    orchestrator: TestOrchestrator,
    mock_registry: MockServiceRegistry,
    ml_config: MLConfig,
}

impl MLTrainingServiceTests {
    /// Initialize ML Training Service test suite
    pub async fn new() -> Result<Self, Box<dyn std::error::Error>> {
        let config = TestFrameworkConfig {
            performance_thresholds: PerformanceThresholds {
                max_e2e_latency_us: 50,      // 50μs end-to-end
                max_order_latency_us: 20,    // 20μs order processing
                max_risk_latency_us: 10,     // 10μs risk validation
                max_ml_latency_ms: 50,       // 50ms ML inference
                max_config_reload_ms: 100,   // 100ms config reload
                min_throughput_ops_sec: 10000, // 10k ops/sec minimum
            },
            database_url: std::env::var("TEST_DATABASE_URL")
                .unwrap_or_else(|_| "postgresql://test:test@localhost:5432/foxhunt_test".to_string()),
            service_ports: {
                let mut ports = HashMap::new();
                ports.insert("trading".to_string(), 50051);
                ports.insert("backtesting".to_string(), 50052);
                ports.insert("ml_training".to_string(), 50053);
                ports
            },
            test_timeout_secs: 180, // ML training may take longer
        };

        let orchestrator = TestOrchestrator::new(config).await?;
        let mock_registry = MockServiceRegistry::new().await?;

        // Load ML configuration
        let config_manager = ConfigManager::new().await?;
        let ml_config = config_manager.get_ml_config().await?;

        Ok(Self {
            orchestrator,
            mock_registry,
            ml_config,
        })
    }

    /// Test Suite 1: Model Training Pipeline
    ///
    /// Validates core ML training functionality:
    /// - Training job initialization
    ///
    /// - Data preprocessing pipeline
    /// - Model training execution
    ///
    /// - Hyperparameter optimization
    /// - Training progress monitoring
    pub async fn test_model_training_pipeline(&self) -> IntegrationTestResult {
        println!("🧠 Testing ML Training Service - Model Training Pipeline");

        let start_time = Instant::now();
        let mut test_results = IntegrationTestResult::new("ML Training Service Model Training Pipeline");

        // Start ML training service
        self.orchestrator.start_service("ml_training").await
            .map_err(|e| format!("Failed to start ML training service: {}", e))?;

        // Wait for service readiness
        tokio::time::sleep(Duration::from_millis(2000)).await;

        // Test Case 1: MAMBA Model Training
        let mamba_training_config = TrainingJobConfig {
            id: uuid::Uuid::new_v4(),
            model_type: ModelType::MAMBA,
            model_name: "MAMBA_EURUSD_Test".to_string(),
            version: "test_v1.0.0".to_string(),
            training_config: TrainingConfig {
                epochs: 5, // Reduced for testing
                batch_size: 32,
                learning_rate: 0.001,
                validation_split: 0.2,
                early_stopping_patience: 3,
                optimizer: "AdamW".to_string(),
                scheduler: Some("CosineAnnealingLR".to_string()),
                regularization: {
                    let mut reg = HashMap::new();
                    reg.insert("dropout".to_string(), 0.1);
                    reg.insert("weight_decay".to_string(), 1e-4);
                    reg
                },
            },
            dataset_config: DatasetConfig {
                symbol: "EURUSD".to_string(),
                timeframe: "1H".to_string(),
                start_date: Utc.ymd_opt(2024, 1, 1).unwrap().and_hms_opt(0, 0, 0).unwrap(),
                end_date: Utc.ymd_opt(2024, 1, 31).unwrap().and_hms_opt(23, 59, 59).unwrap(),
                sequence_length: 100,
                prediction_horizon: 5,
                features: vec![
                    "open".to_string(), "high".to_string(), "low".to_string(), "close".to_string(),
                    "volume".to_string(), "rsi".to_string(), "macd".to_string(),
                ],
                target: "price_direction".to_string(),
            },
            hardware_config: HardwareConfig {
                use_gpu: true,
                gpu_memory_limit: Some(4096), // 4GB
                num_workers: 2,
                pin_memory: true,
            },
        };

        let training_start = Instant::now();
        let training_result = self.orchestrator.start_training_job(mamba_training_config.clone()).await;

        match training_result {
            Ok(job_id) => {
                test_results.add_success("MAMBA training job started successfully");

                // Monitor training progress
                let mut progress_checks = 0;
                let max_checks = 60; // 60 seconds max for test training
                let mut is_complete = false;
                let mut final_metrics: Option<ModelMetrics> = None;

                while progress_checks < max_checks {
                    tokio::time::sleep(Duration::from_millis(1000)).await;

                    match self.orchestrator.get_training_status(&job_id).await {
                        Ok(status) => {
                            match status.status.as_str() {
                                "completed" => {
                                    is_complete = true;
                                    final_metrics = status.metrics;
                                    break;
                                }
                                "failed" | "error" => {
                                    test_results.add_failure(&format!(
                                        "Training failed with status: {} - {}",
                                        status.status,
                                        status.error_message.unwrap_or_else(|| "No error message".to_string())
                                    ));
                                    break;
                                }
                                "training" | "initializing" | "preprocessing" => {
                                    // Log progress
                                    if let Some(progress) = &status.progress {
                                        println!("Training progress: epoch {}/{}, loss: {:.6}",
                                            progress.current_epoch, progress.total_epochs, progress.current_loss);
                                    }
                                }
                                _ => {
                                    test_results.add_failure(&format!("Unknown training status: {}", status.status));
                                    break;
                                }
                            }
                        }
                        Err(e) => {
                            test_results.add_failure(&format!("Failed to get training status: {}", e));
                            break;
                        }
                    }

                    progress_checks += 1;
                }

                if is_complete {
                    let training_duration = training_start.elapsed();
                    test_results.add_success(&format!(
                        "MAMBA training completed successfully in {}s",
                        training_duration.as_secs()
                    ));

                    // Validate training metrics
                    if let Some(metrics) = final_metrics {
                        test_results.add_success("Training metrics available");

                        if metrics.train_loss > 0.0 {
                            test_results.add_success(&format!("Training loss: {:.6}", metrics.train_loss));
                        } else {
                            test_results.add_failure("Invalid training loss");
                        }

                        if let Some(val_loss) = metrics.validation_loss {
                            test_results.add_success(&format!("Validation loss: {:.6}", val_loss));

                            // Check for reasonable validation loss (should be finite and positive)
                            if val_loss.is_finite() && val_loss > 0.0 {
                                test_results.add_success("Validation loss is reasonable");
                            } else {
                                test_results.add_failure("Validation loss is unreasonable");
                            }
                        } else {
                            test_results.add_failure("Validation loss not available");
                        }

                        if let Some(accuracy) = metrics.accuracy {
                            test_results.add_success(&format!("Model accuracy: {:.2}%", accuracy * 100.0));

                            if accuracy > 0.45 { // Better than random for multi-class
                                test_results.add_success("Model shows learning capability");
                            } else {
                                test_results.add_failure("Model accuracy too low for meaningful learning");
                            }
                        } else {
                            test_results.add_failure("Model accuracy not calculated");
                        }

                    } else {
                        test_results.add_failure("Training metrics not available after completion");
                    }

                    // Test Case 2: Model Artifact Validation
                    match self.orchestrator.get_model_artifacts(&job_id).await {
                        Ok(artifacts) => {
                            test_results.add_success("Model artifacts retrieved successfully");

                            if artifacts.model_file_path.is_some() {
                                test_results.add_success("Model file path available");
                            } else {
                                test_results.add_failure("Model file path not available");
                            }

                            if artifacts.config_file_path.is_some() {
                                test_results.add_success("Model config file available");
                            } else {
                                test_results.add_failure("Model config file not available");
                            }

                            if artifacts.metrics_file_path.is_some() {
                                test_results.add_success("Training metrics file available");
                            } else {
                                test_results.add_failure("Training metrics file not available");
                            }
                        }
                        Err(e) => {
                            test_results.add_failure(&format!("Failed to get model artifacts: {}", e));
                        }
                    }

                } else {
                    test_results.add_failure("Training did not complete within timeout");
                }
            }
            Err(e) => {
                test_results.add_failure(&format!("Failed to start MAMBA training job: {}", e));
            }
        }

        test_results.duration = start_time.elapsed();
        test_results.finalize();

        Ok(test_results)
    }

    /// Test Suite 2: Data Processing Pipeline
    ///
    /// Validates data preprocessing and feature engineering:
    /// - Raw data ingestion
    ///
    /// - Feature engineering pipeline
    /// - Data validation and quality checks
    ///
    /// - Dataset splitting and sampling
    /// - Data augmentation techniques
    pub async fn test_data_processing_pipeline(&self) -> IntegrationTestResult {
        println!("📊 Testing ML Training Service - Data Processing Pipeline");

        let start_time = Instant::now();
        let mut test_results = IntegrationTestResult::new("ML Training Service Data Processing Pipeline");

        // Test Case 1: Data Ingestion
        let data_config = DataIngestionConfig {
            symbol: "EURUSD".to_string(),
            timeframe: "1H".to_string(),
            start_date: Utc.ymd_opt(2024, 1, 1).unwrap().and_hms_opt(0, 0, 0).unwrap(),
            end_date: Utc.ymd_opt(2024, 1, 7).unwrap().and_hms_opt(23, 59, 59).unwrap(), // 1 week
            data_sources: vec!["historical_db".to_string(), "live_feed".to_string()],
        };

        let ingestion_start = Instant::now();
        let ingestion_result = self.orchestrator.ingest_training_data(data_config.clone()).await;
        let ingestion_duration = ingestion_start.elapsed();

        match ingestion_result {
            Ok(dataset_id) => {
                test_results.add_success("Data ingestion successful");

                if ingestion_duration.as_secs() <= 10 {
                    test_results.add_success(&format!(
                        "Data ingestion time acceptable: {}s",
                        ingestion_duration.as_secs()
                    ));
                } else {
                    test_results.add_failure(&format!(
                        "Data ingestion time too long: {}s",
                        ingestion_duration.as_secs()
                    ));
                }

                // Validate ingested data
                match self.orchestrator.get_dataset_info(&dataset_id).await {
                    Ok(dataset_info) => {
                        test_results.add_success("Dataset info retrieved");

                        if dataset_info.record_count > 0 {
                            test_results.add_success(&format!("Dataset contains {} records", dataset_info.record_count));
                        } else {
                            test_results.add_failure("Dataset is empty");
                        }

                        if dataset_info.feature_count > 0 {
                            test_results.add_success(&format!("Dataset has {} features", dataset_info.feature_count));
                        } else {
                            test_results.add_failure("Dataset has no features");
                        }

                        // Data quality validation
                        if dataset_info.missing_value_percentage < 0.05 { // Less than 5% missing
                            test_results.add_success(&format!(
                                "Data quality good: {:.2}% missing values",
                                dataset_info.missing_value_percentage * 100.0
                            ));
                        } else {
                            test_results.add_failure(&format!(
                                "Data quality poor: {:.2}% missing values",
                                dataset_info.missing_value_percentage * 100.0
                            ));
                        }
                    }
                    Err(e) => {
                        test_results.add_failure(&format!("Failed to get dataset info: {}", e));
                    }
                }

                // Test Case 2: Feature Engineering
                let feature_config = FeatureEngineeringConfig {
                    base_features: vec!["open", "high", "low", "close", "volume"],
                    technical_indicators: vec![
                        TechnicalIndicator { name: "RSI".to_string(), period: 14, parameters: HashMap::new() },
                        TechnicalIndicator { name: "MACD".to_string(), period: 26, parameters: {
                            let mut params = HashMap::new();
                            params.insert("fast_period".to_string(), 12);
                            params.insert("slow_period".to_string(), 26);
                            params.insert("signal_period".to_string(), 9);
                            params
                        }},
                        TechnicalIndicator { name: "Bollinger_Bands".to_string(), period: 20, parameters: {
                            let mut params = HashMap::new();
                            params.insert("std_dev".to_string(), 2);
                            params
                        }},
                    ],
                    lag_features: vec![1, 2, 3, 5, 10], // Lag periods
                    rolling_stats: vec![
                        RollingStat { window: 5, stat_type: "mean".to_string() },
                        RollingStat { window: 10, stat_type: "std".to_string() },
                        RollingStat { window: 20, stat_type: "min".to_string() },
                        RollingStat { window: 20, stat_type: "max".to_string() },
                    ],
                    normalization_method: "z_score".to_string(),
                };

                let feature_start = Instant::now();
                match self.orchestrator.apply_feature_engineering(&dataset_id, feature_config).await {
                    Ok(engineered_dataset_id) => {
                        let feature_duration = feature_start.elapsed();
                        test_results.add_success("Feature engineering completed");

                        if feature_duration.as_secs() <= 15 {
                            test_results.add_success(&format!(
                                "Feature engineering time acceptable: {}s",
                                feature_duration.as_secs()
                            ));
                        } else {
                            test_results.add_failure(&format!(
                                "Feature engineering time too long: {}s",
                                feature_duration.as_secs()
                            ));
                        }

                        // Validate engineered features
                        match self.orchestrator.get_dataset_info(&engineered_dataset_id).await {
                            Ok(engineered_info) => {
                                if engineered_info.feature_count > dataset_info.feature_count {
                                    test_results.add_success(&format!(
                                        "Features increased: {} -> {}",
                                        dataset_info.feature_count, engineered_info.feature_count
                                    ));
                                } else {
                                    test_results.add_failure("Feature count did not increase");
                                }

                                // Check for NaN/infinite values after feature engineering
                                if engineered_info.has_invalid_values {
                                    test_results.add_failure("Feature engineering introduced invalid values");
                                } else {
                                    test_results.add_success("No invalid values after feature engineering");
                                }
                            }
                            Err(e) => {
                                test_results.add_failure(&format!("Failed to validate engineered features: {}", e));
                            }
                        }
                    }
                    Err(e) => {
                        test_results.add_failure(&format!("Feature engineering failed: {}", e));
                    }
                }

                // Test Case 3: Dataset Splitting
                let split_config = DatasetSplitConfig {
                    train_ratio: 0.7,
                    validation_ratio: 0.15,
                    test_ratio: 0.15,
                    shuffle: true,
                    stratify: true, // For classification targets
                    random_seed: Some(42),
                };

                match self.orchestrator.split_dataset(&dataset_id, split_config).await {
                    Ok(split_result) => {
                        test_results.add_success("Dataset split successful");

                        let total_records = split_result.train_size + split_result.validation_size + split_result.test_size;
                        let expected_records = dataset_info.record_count;

                        if (total_records as f64 - expected_records as f64).abs() / expected_records as f64 < 0.01 {
                            test_results.add_success("Dataset split preserves record count");
                        } else {
                            test_results.add_failure(&format!(
                                "Dataset split lost records: {} -> {}",
                                expected_records, total_records
                            ));
                        }

                        // Validate split ratios
                        let actual_train_ratio = split_result.train_size as f64 / total_records as f64;
                        let actual_val_ratio = split_result.validation_size as f64 / total_records as f64;
                        let actual_test_ratio = split_result.test_size as f64 / total_records as f64;

                        if (actual_train_ratio - split_config.train_ratio).abs() < 0.05 {
                            test_results.add_success(&format!(
                                "Train split ratio correct: {:.2}%",
                                actual_train_ratio * 100.0
                            ));
                        } else {
                            test_results.add_failure(&format!(
                                "Train split ratio incorrect: expected {:.2}%, got {:.2}%",
                                split_config.train_ratio * 100.0, actual_train_ratio * 100.0
                            ));
                        }
                    }
                    Err(e) => {
                        test_results.add_failure(&format!("Dataset splitting failed: {}", e));
                    }
                }
            }
            Err(e) => {
                test_results.add_failure(&format!("Data ingestion failed: {}", e));
            }
        }

        test_results.duration = start_time.elapsed();
        test_results.finalize();

        Ok(test_results)
    }

    /// Test Suite 3: Model Validation and Metrics
    ///
    /// Validates model evaluation and metrics calculation:
    /// - Cross-validation procedures
    ///
    /// - Performance metrics calculation
    /// - Model comparison and selection
    ///
    /// - Statistical significance testing
    pub async fn test_model_validation_and_metrics(&self) -> IntegrationTestResult {
        println!("📈 Testing ML Training Service - Model Validation and Metrics");

        let start_time = Instant::now();
        let mut test_results = IntegrationTestResult::new("ML Training Service Model Validation and Metrics");

        // Test Case 1: Cross-Validation Setup
        let cv_config = CrossValidationConfig {
            method: "time_series".to_string(), // Appropriate for financial data
            n_folds: 5,
            validation_window: 720, // 30 days of hourly data
            step_size: 168, // 1 week step
            gap_size: 24, // 1 day gap to prevent look-ahead
        };

        let model_configs = vec![
            ModelConfig {
                model_type: ModelType::MAMBA,
                name: "MAMBA_CV_Test".to_string(),
                hyperparameters: {
                    let mut params = HashMap::new();
                    params.insert("hidden_dim".to_string(), serde_json::Value::from(256));
                    params.insert("n_layers".to_string(), serde_json::Value::from(4));
                    params.insert("dropout".to_string(), serde_json::Value::from(0.1));
                    params
                },
            },
            ModelConfig {
                model_type: ModelType::TFT,
                name: "TFT_CV_Test".to_string(),
                hyperparameters: {
                    let mut params = HashMap::new();
                    params.insert("hidden_size".to_string(), serde_json::Value::from(128));
                    params.insert("attention_heads".to_string(), serde_json::Value::from(4));
                    params.insert("dropout".to_string(), serde_json::Value::from(0.1));
                    params
                },
            },
        ];

        let cv_start = Instant::now();
        match self.orchestrator.run_cross_validation(model_configs, cv_config).await {
            Ok(cv_job_id) => {
                test_results.add_success("Cross-validation job started");

                // Monitor CV progress
                let mut cv_checks = 0;
                let max_cv_checks = 120; // 2 minutes for CV
                let mut cv_complete = false;
                let mut cv_results: Option<CrossValidationResults> = None;

                while cv_checks < max_cv_checks {
                    tokio::time::sleep(Duration::from_millis(1000)).await;

                    match self.orchestrator.get_cv_status(&cv_job_id).await {
                        Ok(status) => {
                            match status.status.as_str() {
                                "completed" => {
                                    cv_complete = true;
                                    cv_results = status.results;
                                    break;
                                }
                                "failed" | "error" => {
                                    test_results.add_failure(&format!("Cross-validation failed: {}", status.status));
                                    break;
                                }
                                "running" => {
                                    if let Some(progress) = &status.progress {
                                        println!("CV Progress: fold {}/{}, model {}/{}",
                                            progress.current_fold, progress.total_folds,
                                            progress.current_model, progress.total_models);
                                    }
                                }
                                _ => {
                                    // Continue waiting
                                }
                            }
                        }
                        Err(e) => {
                            test_results.add_failure(&format!("Failed to get CV status: {}", e));
                            break;
                        }
                    }

                    cv_checks += 1;
                }

                if cv_complete {
                    let cv_duration = cv_start.elapsed();
                    test_results.add_success(&format!(
                        "Cross-validation completed in {}s",
                        cv_duration.as_secs()
                    ));

                    if let Some(results) = cv_results {
                        test_results.add_success("Cross-validation results available");

                        // Validate CV results for each model
                        for model_result in &results.model_results {
                            test_results.add_success(&format!(
                                "Model {} CV completed", model_result.model_name
                            ));

                            if model_result.mean_score > 0.0 {
                                test_results.add_success(&format!(
                                    "{} mean CV score: {:.4} ± {:.4}",
                                    model_result.model_name,
                                    model_result.mean_score,
                                    model_result.std_score
                                ));
                            } else {
                                test_results.add_failure(&format!(
                                    "{} invalid mean CV score: {}",
                                    model_result.model_name, model_result.mean_score
                                ));
                            }

                            // Check for reasonable standard deviation
                            if model_result.std_score > 0.0 && model_result.std_score < model_result.mean_score {
                                test_results.add_success(&format!(
                                    "{} reasonable score variance", model_result.model_name
                                ));
                            } else if model_result.std_score == 0.0 {
                                test_results.add_failure(&format!(
                                    "{} zero score variance (suspicious)", model_result.model_name
                                ));
                            } else {
                                test_results.add_failure(&format!(
                                    "{} high score variance: {:.4}", model_result.model_name, model_result.std_score
                                ));
                            }
                        }

                        // Test Case 2: Statistical Significance Testing
                        if results.model_results.len() >= 2 {
                            match self.orchestrator.perform_statistical_test(&cv_job_id, "paired_t_test").await {
                                Ok(stat_results) => {
                                    test_results.add_success("Statistical significance testing completed");

                                    if let Some(p_value) = stat_results.p_value {
                                        test_results.add_success(&format!("Statistical test p-value: {:.4}", p_value));

                                        if p_value < 0.05 {
                                            test_results.add_success("Significant difference between models detected");
                                        } else {
                                            test_results.add_success("No significant difference between models");
                                        }
                                    } else {
                                        test_results.add_failure("Statistical test p-value not available");
                                    }
                                }
                                Err(e) => {
                                    test_results.add_failure(&format!("Statistical testing failed: {}", e));
                                }
                            }
                        }

                        // Test Case 3: Best Model Selection
                        if let Some(best_model) = &results.best_model {
                            test_results.add_success(&format!("Best model selected: {}", best_model.model_name));

                            if best_model.final_score > 0.0 {
                                test_results.add_success(&format!(
                                    "Best model final score: {:.4}", best_model.final_score
                                ));
                            } else {
                                test_results.add_failure("Best model has invalid final score");
                            }
                        } else {
                            test_results.add_failure("No best model selected");
                        }

                    } else {
                        test_results.add_failure("Cross-validation results not available");
                    }

                } else {
                    test_results.add_failure("Cross-validation did not complete within timeout");
                }
            }
            Err(e) => {
                test_results.add_failure(&format!("Failed to start cross-validation: {}", e));
            }
        }

        // Test Case 4: Comprehensive Metrics Calculation
        let test_model_config = ModelConfig {
            model_type: ModelType::DQN,
            name: "DQN_Metrics_Test".to_string(),
            hyperparameters: {
                let mut params = HashMap::new();
                params.insert("hidden_layers".to_string(), serde_json::Value::from(vec![256, 128]));
                params.insert("learning_rate".to_string(), serde_json::Value::from(0.001));
                params
            },
        };

        let quick_training_config = TrainingJobConfig {
            id: uuid::Uuid::new_v4(),
            model_type: ModelType::DQN,
            model_name: "DQN_Metrics_Test".to_string(),
            version: "test_metrics_v1".to_string(),
            training_config: TrainingConfig {
                epochs: 3, // Very quick training for metrics testing
                batch_size: 64,
                learning_rate: 0.001,
                validation_split: 0.2,
                early_stopping_patience: 2,
                optimizer: "Adam".to_string(),
                scheduler: None,
                regularization: HashMap::new(),
            },
            dataset_config: DatasetConfig {
                symbol: "EURUSD".to_string(),
                timeframe: "1H".to_string(),
                start_date: Utc.ymd_opt(2024, 1, 1).unwrap().and_hms_opt(0, 0, 0).unwrap(),
                end_date: Utc.ymd_opt(2024, 1, 7).unwrap().and_hms_opt(23, 59, 59).unwrap(),
                sequence_length: 50,
                prediction_horizon: 1,
                features: vec!["close".to_string(), "volume".to_string()],
                target: "price_direction".to_string(),
            },
            hardware_config: HardwareConfig {
                use_gpu: false, // CPU for quick testing
                gpu_memory_limit: None,
                num_workers: 1,
                pin_memory: false,
            },
        };

        match self.orchestrator.start_training_job(quick_training_config).await {
            Ok(metrics_job_id) => {
                // Wait for quick training completion
                tokio::time::sleep(Duration::from_secs(30)).await;

                match self.orchestrator.calculate_comprehensive_metrics(&metrics_job_id).await {
                    Ok(comprehensive_metrics) => {
                        test_results.add_success("Comprehensive metrics calculated");

                        // Validate presence of key metrics
                        if comprehensive_metrics.accuracy.is_some() {
                            test_results.add_success("Accuracy metric available");
                        } else {
                            test_results.add_failure("Accuracy metric missing");
                        }

                        if comprehensive_metrics.precision.is_some() {
                            test_results.add_success("Precision metric available");
                        } else {
                            test_results.add_failure("Precision metric missing");
                        }

                        if comprehensive_metrics.recall.is_some() {
                            test_results.add_success("Recall metric available");
                        } else {
                            test_results.add_failure("Recall metric missing");
                        }

                        if comprehensive_metrics.f1_score.is_some() {
                            test_results.add_success("F1-score metric available");
                        } else {
                            test_results.add_failure("F1-score metric missing");
                        }

                        if comprehensive_metrics.auc_roc.is_some() {
                            test_results.add_success("AUC-ROC metric available");
                        } else {
                            test_results.add_failure("AUC-ROC metric missing");
                        }

                        // Financial-specific metrics
                        if comprehensive_metrics.sharpe_ratio.is_some() {
                            test_results.add_success("Sharpe ratio available");
                        } else {
                            test_results.add_failure("Sharpe ratio missing");
                        }

                        if comprehensive_metrics.max_drawdown.is_some() {
                            test_results.add_success("Max drawdown available");
                        } else {
                            test_results.add_failure("Max drawdown missing");
                        }
                    }
                    Err(e) => {
                        test_results.add_failure(&format!("Failed to calculate comprehensive metrics: {}", e));
                    }
                }
            }
            Err(e) => {
                test_results.add_failure(&format!("Failed to start metrics test training: {}", e));
            }
        }

        test_results.duration = start_time.elapsed();
        test_results.finalize();

        Ok(test_results)
    }

    /// Test Suite 4: Model Deployment Pipeline
    ///
    /// Validates model deployment and serving:
    /// - Model versioning and registry
    ///
    /// - Deployment to serving infrastructure
    /// - A/B testing setup
    ///
    /// - Model monitoring and alerting
    pub async fn test_model_deployment_pipeline(&self) -> IntegrationTestResult {
        println!("🚀 Testing ML Training Service - Model Deployment Pipeline");

        let start_time = Instant::now();
        let mut test_results = IntegrationTestResult::new("ML Training Service Model Deployment Pipeline");

        // Test Case 1: Model Registration
        let model_version = ModelVersion {
            id: uuid::Uuid::new_v4(),
            name: "MAMBA_EURUSD_Deploy_Test".to_string(),
            version: "v1.0.0".to_string(),
            model_type: ModelType::MAMBA,
            training_job_id: uuid::Uuid::new_v4(),
            performance_metrics: {
                let mut metrics = HashMap::new();
                metrics.insert("accuracy".to_string(), 0.74);
                metrics.insert("precision".to_string(), 0.71);
                metrics.insert("recall".to_string(), 0.78);
                metrics.insert("f1_score".to_string(), 0.74);
                metrics.insert("sharpe_ratio".to_string(), 1.85);
                metrics
            },
            artifacts_path: "/models/mamba_eurusd_v1.0.0/".to_string(),
            created_at: Utc::now(),
            is_active: false,
            deployment_status: "pending".to_string(),
        };

        let registration_start = Instant::now();
        match self.orchestrator.register_model_version(model_version.clone()).await {
            Ok(registration_id) => {
                let registration_duration = registration_start.elapsed();
                test_results.add_success("Model version registered successfully");

                if registration_duration.as_millis() <= 1000 {
                    test_results.add_success(&format!(
                        "Model registration time acceptable: {}ms",
                        registration_duration.as_millis()
                    ));
                } else {
                    test_results.add_failure(&format!(
                        "Model registration time too long: {}ms",
                        registration_duration.as_millis()
                    ));
                }

                // Test Case 2: Model Deployment
                let deployment_config = DeploymentConfig {
                    target_environment: "staging".to_string(),
                    serving_config: ServingConfig {
                        max_batch_size: 32,
                        max_latency_ms: 50, // 50ms inference latency requirement
                        auto_scaling: true,
                        min_replicas: 1,
                        max_replicas: 3,
                        cpu_limit: "1000m".to_string(),
                        memory_limit: "2Gi".to_string(),
                    },
                    rollout_strategy: RolloutStrategy {
                        strategy_type: "blue_green".to_string(),
                        traffic_split: 0.1, // 10% initial traffic
                        rollback_threshold: 0.05, // 5% error rate triggers rollback
                        monitoring_duration_minutes: 10,
                    },
                };

                let deployment_start = Instant::now();
                match self.orchestrator.deploy_model(&registration_id, deployment_config).await {
                    Ok(deployment_id) => {
                        let deployment_duration = deployment_start.elapsed();
                        test_results.add_success("Model deployment initiated");

                        if deployment_duration.as_secs() <= 30 {
                            test_results.add_success(&format!(
                                "Deployment initiation time acceptable: {}s",
                                deployment_duration.as_secs()
                            ));
                        } else {
                            test_results.add_failure(&format!(
                                "Deployment initiation time too long: {}s",
                                deployment_duration.as_secs()
                            ));
                        }

                        // Monitor deployment progress
                        let mut deployment_checks = 0;
                        let max_deployment_checks = 60; // 1 minute for staging deployment
                        let mut deployment_complete = false;

                        while deployment_checks < max_deployment_checks {
                            tokio::time::sleep(Duration::from_millis(1000)).await;

                            match self.orchestrator.get_deployment_status(&deployment_id).await {
                                Ok(status) => {
                                    match status.status.as_str() {
                                        "deployed" | "active" => {
                                            deployment_complete = true;
                                            test_results.add_success("Model deployed successfully");
                                            break;
                                        }
                                        "failed" | "error" => {
                                            test_results.add_failure(&format!(
                                                "Deployment failed: {}",
                                                status.error_message.unwrap_or_else(|| "No error message".to_string())
                                            ));
                                            break;
                                        }
                                        "deploying" | "initializing" => {
                                            // Continue waiting
                                        }
                                        _ => {
                                            test_results.add_failure(&format!("Unknown deployment status: {}", status.status));
                                            break;
                                        }
                                    }
                                }
                                Err(e) => {
                                    test_results.add_failure(&format!("Failed to get deployment status: {}", e));
                                    break;
                                }
                            }

                            deployment_checks += 1;
                        }

                        if deployment_complete {
                            // Test Case 3: Model Serving Validation
                            let test_inference_data = InferenceRequest {
                                model_version_id: registration_id.clone(),
                                input_data: vec![
                                    1.0650, 1.0655, 1.0648, 1.0653, // OHLC
                                    1500.0, // Volume
                                    65.5,   // RSI
                                    0.0012, // MACD
                                ],
                                request_id: uuid::Uuid::new_v4(),
                                timestamp: Utc::now(),
                            };

                            let inference_start = Instant::now();
                            match self.orchestrator.test_model_inference(test_inference_data).await {
                                Ok(inference_result) => {
                                    let inference_duration = inference_start.elapsed();
                                    test_results.add_success("Model inference test successful");

                                    if inference_duration.as_millis() <= self.orchestrator.config.performance_thresholds.max_ml_latency_ms {
                                        test_results.add_success(&format!(
                                            "Inference latency within threshold: {}ms <= {}ms",
                                            inference_duration.as_millis(),
                                            self.orchestrator.config.performance_thresholds.max_ml_latency_ms
                                        ));
                                    } else {
                                        test_results.add_failure(&format!(
                                            "Inference latency exceeds threshold: {}ms > {}ms",
                                            inference_duration.as_millis(),
                                            self.orchestrator.config.performance_thresholds.max_ml_latency_ms
                                        ));
                                    }

                                    // Validate inference result structure
                                    if inference_result.prediction.is_some() {
                                        test_results.add_success("Model prediction available");
                                    } else {
                                        test_results.add_failure("Model prediction missing");
                                    }

                                    if inference_result.confidence.is_some() {
                                        let confidence = inference_result.confidence.unwrap();
                                        if confidence >= 0.0 && confidence <= 1.0 {
                                            test_results.add_success(&format!(
                                                "Model confidence valid: {:.3}", confidence
                                            ));
                                        } else {
                                            test_results.add_failure(&format!(
                                                "Model confidence out of range: {:.3}", confidence
                                            ));
                                        }
                                    } else {
                                        test_results.add_failure("Model confidence missing");
                                    }
                                }
                                Err(e) => {
                                    test_results.add_failure(&format!("Model inference test failed: {}", e));
                                }
                            }

                            // Test Case 4: Model Monitoring Setup
                            let monitoring_config = ModelMonitoringConfig {
                                metrics_to_track: vec![
                                    "inference_latency".to_string(),
                                    "prediction_accuracy".to_string(),
                                    "data_drift".to_string(),
                                    "model_drift".to_string(),
                                ],
                                alert_thresholds: {
                                    let mut thresholds = HashMap::new();
                                    thresholds.insert("inference_latency_p95".to_string(), 100.0); // 100ms
                                    thresholds.insert("accuracy_drop".to_string(), 0.05); // 5% drop
                                    thresholds.insert("data_drift_score".to_string(), 0.1);
                                    thresholds
                                },
                                monitoring_frequency_minutes: 5,
                                retention_days: 30,
                            };

                            match self.orchestrator.setup_model_monitoring(&deployment_id, monitoring_config).await {
                                Ok(_) => {
                                    test_results.add_success("Model monitoring setup successful");
                                }
                                Err(e) => {
                                    test_results.add_failure(&format!("Model monitoring setup failed: {}", e));
                                }
                            }

                        } else {
                            test_results.add_failure("Model deployment did not complete within timeout");
                        }
                    }
                    Err(e) => {
                        test_results.add_failure(&format!("Failed to deploy model: {}", e));
                    }
                }
            }
            Err(e) => {
                test_results.add_failure(&format!("Failed to register model version: {}", e));
            }
        }

        test_results.duration = start_time.elapsed();
        test_results.finalize();

        Ok(test_results)
    }

    /// Execute complete ML Training Service test suite
    pub async fn run_all_tests(&self) -> Result<Vec<IntegrationTestResult>, Box<dyn std::error::Error>> {
        println!("🚀 Starting ML Training Service Integration Test Suite");

        let mut results = Vec::new();

        // Test Suite 1: Model Training Pipeline
        match timeout(Duration::from_secs(self.orchestrator.config.test_timeout_secs * 2),
                     self.test_model_training_pipeline()).await {
            Ok(Ok(result)) => results.push(result),
            Ok(Err(e)) => {
                let mut failed_result = IntegrationTestResult::new("ML Training Service Model Training Pipeline");
                failed_result.add_failure(&format!("Test suite failed: {}", e));
                failed_result.finalize();
                results.push(failed_result);
            },
            Err(_) => {
                let mut timeout_result = IntegrationTestResult::new("ML Training Service Model Training Pipeline");
                timeout_result.add_failure("Test suite timed out");
                timeout_result.finalize();
                results.push(timeout_result);
            }
        }

        // Test Suite 2: Data Processing Pipeline
        match timeout(Duration::from_secs(self.orchestrator.config.test_timeout_secs),
                     self.test_data_processing_pipeline()).await {
            Ok(Ok(result)) => results.push(result),
            Ok(Err(e)) => {
                let mut failed_result = IntegrationTestResult::new("ML Training Service Data Processing Pipeline");
                failed_result.add_failure(&format!("Test suite failed: {}", e));
                failed_result.finalize();
                results.push(failed_result);
            },
            Err(_) => {
                let mut timeout_result = IntegrationTestResult::new("ML Training Service Data Processing Pipeline");
                timeout_result.add_failure("Test suite timed out");
                timeout_result.finalize();
                results.push(timeout_result);
            }
        }

        // Test Suite 3: Model Validation and Metrics
        match timeout(Duration::from_secs(self.orchestrator.config.test_timeout_secs * 3),
                     self.test_model_validation_and_metrics()).await {
            Ok(Ok(result)) => results.push(result),
            Ok(Err(e)) => {
                let mut failed_result = IntegrationTestResult::new("ML Training Service Model Validation and Metrics");
                failed_result.add_failure(&format!("Test suite failed: {}", e));
                failed_result.finalize();
                results.push(failed_result);
            },
            Err(_) => {
                let mut timeout_result = IntegrationTestResult::new("ML Training Service Model Validation and Metrics");
                timeout_result.add_failure("Test suite timed out");
                timeout_result.finalize();
                results.push(timeout_result);
            }
        }

        // Test Suite 4: Model Deployment Pipeline
        match timeout(Duration::from_secs(self.orchestrator.config.test_timeout_secs * 2),
                     self.test_model_deployment_pipeline()).await {
            Ok(Ok(result)) => results.push(result),
            Ok(Err(e)) => {
                let mut failed_result = IntegrationTestResult::new("ML Training Service Model Deployment Pipeline");
                failed_result.add_failure(&format!("Test suite failed: {}", e));
                failed_result.finalize();
                results.push(failed_result);
            },
            Err(_) => {
                let mut timeout_result = IntegrationTestResult::new("ML Training Service Model Deployment Pipeline");
                timeout_result.add_failure("Test suite timed out");
                timeout_result.finalize();
                results.push(timeout_result);
            }
        }

        // Print summary
        let total_tests = results.len();
        let passed_tests = results.iter().filter(|r| r.passed).count();
        let failed_tests = total_tests - passed_tests;

        println!("📊 ML Training Service Integration Test Summary:");
        println!("   Total Test Suites: {}", total_tests);
        println!("   Passed: {} ✅", passed_tests);
        println!("   Failed: {} ❌", failed_tests);

        if failed_tests == 0 {
            println!("🎉 All ML Training Service integration tests passed!");
        } else {
            println!("⚠️ {} ML Training Service integration test suite(s) failed", failed_tests);
        }

        Ok(results)
    }
}

// Supporting types for ML training tests
#[derive(Debug, Clone)]
pub struct TrainingJobConfig {
    pub id: uuid::Uuid,
    pub model_type: ModelType,
    pub model_name: String,
    pub version: String,
    pub training_config: TrainingConfig,
    pub dataset_config: DatasetConfig,
    pub hardware_config: HardwareConfig,
}

#[derive(Debug, Clone)]
pub struct DatasetConfig {
    pub symbol: String,
    pub timeframe: String,
    pub start_date: DateTime<Utc>,
    pub end_date: DateTime<Utc>,
    pub sequence_length: usize,
    pub prediction_horizon: usize,
    pub features: Vec<String>,
    pub target: String,
}

#[derive(Debug, Clone)]
pub struct HardwareConfig {
    pub use_gpu: bool,
    pub gpu_memory_limit: Option<u32>,
    pub num_workers: usize,
    pub pin_memory: bool,
}

// Additional supporting types...
#[derive(Debug, Clone)]
pub struct DataIngestionConfig {
    pub symbol: String,
    pub timeframe: String,
    pub start_date: DateTime<Utc>,
    pub end_date: DateTime<Utc>,
    pub data_sources: Vec<String>,
}

#[derive(Debug, Clone)]
pub struct FeatureEngineeringConfig {
    pub base_features: Vec<&'static str>,
    pub technical_indicators: Vec<TechnicalIndicator>,
    pub lag_features: Vec<usize>,
    pub rolling_stats: Vec<RollingStat>,
    pub normalization_method: String,
}

#[derive(Debug, Clone)]
pub struct TechnicalIndicator {
    pub name: String,
    pub period: usize,
    pub parameters: HashMap<String, i32>,
}

#[derive(Debug, Clone)]
pub struct RollingStat {
    pub window: usize,
    pub stat_type: String,
}

#[derive(Debug, Clone)]
pub struct DatasetSplitConfig {
    pub train_ratio: f64,
    pub validation_ratio: f64,
    pub test_ratio: f64,
    pub shuffle: bool,
    pub stratify: bool,
    pub random_seed: Option<u64>,
}

#[derive(Debug, Clone)]
pub struct CrossValidationConfig {
    pub method: String,
    pub n_folds: usize,
    pub validation_window: usize,
    pub step_size: usize,
    pub gap_size: usize,
}

#[derive(Debug, Clone)]
pub struct CrossValidationResults {
    pub model_results: Vec<ModelCVResult>,
    pub best_model: Option<BestModelResult>,
}

#[derive(Debug, Clone)]
pub struct ModelCVResult {
    pub model_name: String,
    pub mean_score: f64,
    pub std_score: f64,
    pub fold_scores: Vec<f64>,
}

#[derive(Debug, Clone)]
pub struct BestModelResult {
    pub model_name: String,
    pub final_score: f64,
    pub selection_criteria: String,
}

#[derive(Debug, Clone)]
pub struct DeploymentConfig {
    pub target_environment: String,
    pub serving_config: ServingConfig,
    pub rollout_strategy: RolloutStrategy,
}

#[derive(Debug, Clone)]
pub struct ServingConfig {
    pub max_batch_size: usize,
    pub max_latency_ms: u64,
    pub auto_scaling: bool,
    pub min_replicas: usize,
    pub max_replicas: usize,
    pub cpu_limit: String,
    pub memory_limit: String,
}

#[derive(Debug, Clone)]
pub struct RolloutStrategy {
    pub strategy_type: String,
    pub traffic_split: f64,
    pub rollback_threshold: f64,
    pub monitoring_duration_minutes: usize,
}

#[derive(Debug, Clone)]
pub struct InferenceRequest {
    pub model_version_id: uuid::Uuid,
    pub input_data: Vec<f64>,
    pub request_id: uuid::Uuid,
    pub timestamp: DateTime<Utc>,
}

#[derive(Debug, Clone)]
pub struct ModelMonitoringConfig {
    pub metrics_to_track: Vec<String>,
    pub alert_thresholds: HashMap<String, f64>,
    pub monitoring_frequency_minutes: usize,
    pub retention_days: usize,
}

#[cfg(test)]
mod tests {
    use super::*;
    use tokio;

    #[tokio::test]
    async fn integration_test_ml_training_service_complete() {
        let test_suite = MLTrainingServiceTests::new().await
            .expect("Failed to initialize ML Training Service test suite");

        let results = test_suite.run_all_tests().await
            .expect("Failed to run ML Training Service test suite");

        // Ensure all tests passed
        for result in &results {
            assert!(result.passed, "Test suite '{}' failed: {:?}", result.test_name, result.failures);
        }

        // Validate performance requirements met (longer timeouts for ML operations)
        for result in &results {
            assert!(
                result.duration.as_secs() <= 600, // 10 minutes max for ML training tests
                "Test suite '{}' took too long: {}s",
                result.test_name,
                result.duration.as_secs()
            );
        }
    }
}