foxhunt/tests/ml_monitoring_integration.rs

//! Comprehensive Integration Tests for ML Monitoring System (Wave 68 Agent 3)
//!
//! Tests the MLPerformanceMonitor, MLFallbackManager, and MLMetricsCollector
//! integration from Wave 67 Agent 1.
//!
//! Validates:
//! - 12 Prometheus metrics recording correctly
//! - 6 alert types with subscription handlers
//! - Performance overhead <10μs
//! - Failover and circuit breaker integration
//! - Cross-component integration

use std::time::{Duration, Instant, SystemTime};
use tokio::time::sleep;

// Import the monitoring components from trading_service
// Note: These are in services/trading_service/src/services/
// We'll use conditional compilation or test helpers

#[cfg(test)]
mod ml_monitoring_tests {
    use super::*;

    // ==================================================================================
    // Test Suite 1: MLPerformanceMonitor Alert System
    // ==================================================================================

    #[tokio::test]
    async fn test_alert_subscription_handler() {
        // Create monitor with default config
        let monitor = create_test_monitor().await;

        // Subscribe to alerts
        let mut alert_receiver = monitor.subscribe_alerts();

        // Record a sample that triggers latency alert
        let sample = create_high_latency_sample("test_model", 5000); // 5ms > 1ms threshold
        monitor.record_sample(sample).await;

        // Wait for alert to be broadcast
        let alert_result = tokio::time::timeout(Duration::from_millis(100), alert_receiver.recv()).await;

        assert!(alert_result.is_ok(), "Alert should be received within timeout");
        let alert = alert_result.unwrap().unwrap();

        assert_eq!(alert.model_id, "test_model");
        assert_eq!(alert.alert_type, AlertType::HighLatency);
        assert_eq!(alert.severity, AlertSeverity::Warning);
        assert!(alert.current_value > 1000.0, "Latency should exceed threshold");
    }

    #[tokio::test]
    async fn test_multiple_subscribers_receive_alerts() {
        let monitor = create_test_monitor().await;

        // Create 3 subscribers
        let mut subscriber1 = monitor.subscribe_alerts();
        let mut subscriber2 = monitor.subscribe_alerts();
        let mut subscriber3 = monitor.subscribe_alerts();

        // Trigger alert
        let sample = create_high_latency_sample("multi_test", 2000);
        monitor.record_sample(sample).await;

        // All subscribers should receive the alert
        let results = tokio::join!(
            tokio::time::timeout(Duration::from_millis(100), subscriber1.recv()),
            tokio::time::timeout(Duration::from_millis(100), subscriber2.recv()),
            tokio::time::timeout(Duration::from_millis(100), subscriber3.recv()),
        );

        assert!(results.0.is_ok(), "Subscriber 1 should receive alert");
        assert!(results.1.is_ok(), "Subscriber 2 should receive alert");
        assert!(results.2.is_ok(), "Subscriber 3 should receive alert");

        // Verify all alerts are identical
        let alert1 = results.0.unwrap().unwrap();
        let alert2 = results.1.unwrap().unwrap();
        let alert3 = results.2.unwrap().unwrap();

        assert_eq!(alert1.alert_id, alert2.alert_id);
        assert_eq!(alert2.alert_id, alert3.alert_id);
    }

    #[tokio::test]
    async fn test_latency_alert_generation() {
        let mut config = AlertConfig::default();
        config.latency_threshold_us = 500; // 500μs threshold
        config.enable_latency_alerts = true;

        let monitor = create_monitor_with_config(config).await;
        let mut receiver = monitor.subscribe_alerts();

        // Record sample below threshold - no alert
        let sample1 = create_sample_with_latency("model_a", 300);
        monitor.record_sample(sample1).await;

        let no_alert = tokio::time::timeout(Duration::from_millis(50), receiver.recv()).await;
        assert!(no_alert.is_err(), "No alert should be generated for latency below threshold");

        // Record sample above threshold - should trigger alert
        let sample2 = create_sample_with_latency("model_a", 1000);
        monitor.record_sample(sample2).await;

        let alert_result = tokio::time::timeout(Duration::from_millis(100), receiver.recv()).await;
        assert!(alert_result.is_ok(), "Alert should be generated for high latency");

        let alert = alert_result.unwrap().unwrap();
        assert_eq!(alert.alert_type, AlertType::HighLatency);
        assert!(alert.current_value >= 500.0);
    }

    #[tokio::test]
    async fn test_accuracy_alert_generation() {
        let mut config = AlertConfig::default();
        config.accuracy_threshold = 0.7;
        config.enable_accuracy_alerts = true;

        let monitor = create_monitor_with_config(config).await;
        let mut receiver = monitor.subscribe_alerts();

        // Record correct prediction - no alert
        let sample1 = create_sample_with_accuracy("model_b", true);
        monitor.record_sample(sample1).await;

        let no_alert = tokio::time::timeout(Duration::from_millis(50), receiver.recv()).await;
        assert!(no_alert.is_err(), "No alert for correct prediction");

        // Record incorrect prediction - should trigger alert
        let sample2 = create_sample_with_accuracy("model_b", false);
        monitor.record_sample(sample2).await;

        let alert_result = tokio::time::timeout(Duration::from_millis(100), receiver.recv()).await;
        assert!(alert_result.is_ok(), "Alert should be generated for low accuracy");

        let alert = alert_result.unwrap().unwrap();
        assert_eq!(alert.alert_type, AlertType::LowAccuracy);
        assert_eq!(alert.severity, AlertSeverity::Critical);
    }

    #[tokio::test]
    async fn test_memory_alert_generation() {
        let mut config = AlertConfig::default();
        config.memory_threshold_mb = 256.0;
        config.enable_memory_alerts = true;

        let monitor = create_monitor_with_config(config).await;
        let mut receiver = monitor.subscribe_alerts();

        // Low memory usage - no alert
        let sample1 = create_sample_with_memory("model_c", 128.0);
        monitor.record_sample(sample1).await;

        let no_alert = tokio::time::timeout(Duration::from_millis(50), receiver.recv()).await;
        assert!(no_alert.is_err(), "No alert for normal memory usage");

        // High memory usage - should trigger alert
        let sample2 = create_sample_with_memory("model_c", 512.0);
        monitor.record_sample(sample2).await;

        let alert_result = tokio::time::timeout(Duration::from_millis(100), receiver.recv()).await;
        assert!(alert_result.is_ok(), "Alert should be generated for high memory");

        let alert = alert_result.unwrap().unwrap();
        assert_eq!(alert.alert_type, AlertType::HighMemoryUsage);
        assert!(alert.current_value >= 256.0);
    }

    #[tokio::test]
    async fn test_drift_detection_alert() {
        let mut config = AlertConfig::default();
        config.enable_drift_detection = true;
        config.drift_window_size = 20; // Smaller window for testing
        config.drift_threshold_percent = 15.0;

        let drift_threshold = config.drift_threshold_percent; // Save before move
        let monitor = create_monitor_with_config(config).await;
        let mut receiver = monitor.subscribe_alerts();

        // Record 10 high-accuracy samples
        for i in 0..10 {
            let sample = create_sample_with_accuracy(&format!("drift_model_{}", i % 2), true);
            monitor.record_sample(sample).await;
        }

        // Record 10 low-accuracy samples to trigger drift
        for i in 0..10 {
            let sample = create_sample_with_accuracy(&format!("drift_model_{}", i % 2), false);
            monitor.record_sample(sample).await;
        }

        // Wait for drift alert
        let alert_result = tokio::time::timeout(Duration::from_millis(200), receiver.recv()).await;

        if let Ok(Ok(alert)) = alert_result {
            assert_eq!(alert.alert_type, AlertType::ModelDrift);
            assert_eq!(alert.severity, AlertSeverity::Critical);
            assert!(alert.current_value >= drift_threshold);
        }
        // Note: Drift detection may not trigger if window not filled properly
    }

    #[tokio::test]
    async fn test_alert_cooldown_enforcement() {
        let mut config = AlertConfig::default();
        config.latency_threshold_us = 100;
        config.alert_cooldown_seconds = 2; // 2 second cooldown
        config.enable_latency_alerts = true;

        let monitor = create_monitor_with_config(config).await;
        let mut receiver = monitor.subscribe_alerts();

        // First alert should be generated
        let sample1 = create_sample_with_latency("cooldown_test", 500);
        monitor.record_sample(sample1).await;

        let alert1 = tokio::time::timeout(Duration::from_millis(100), receiver.recv()).await;
        assert!(alert1.is_ok(), "First alert should be generated");

        // Second alert within cooldown - should NOT be generated
        let sample2 = create_sample_with_latency("cooldown_test", 500);
        monitor.record_sample(sample2).await;

        let alert2 = tokio::time::timeout(Duration::from_millis(100), receiver.recv()).await;
        assert!(alert2.is_err(), "Second alert should be suppressed by cooldown");

        // Wait for cooldown to expire
        sleep(Duration::from_secs(3)).await;

        // Third alert after cooldown - should be generated
        let sample3 = create_sample_with_latency("cooldown_test", 500);
        monitor.record_sample(sample3).await;

        let alert3 = tokio::time::timeout(Duration::from_millis(100), receiver.recv()).await;
        assert!(alert3.is_ok(), "Alert should be generated after cooldown expires");
    }

    #[tokio::test]
    async fn test_statistics_calculation_accuracy() {
        let monitor = create_test_monitor().await;

        // Record 100 samples with known values
        for i in 0..100 {
            let latency = 100 + (i * 10); // 100, 110, 120, ... 1090 μs
            let accuracy = if i < 75 { 1.0 } else { 0.0 }; // 75% accuracy

            let sample = create_sample("stats_model", latency, accuracy > 0.5);
            monitor.record_sample(sample).await;
        }

        // Get statistics
        let stats = monitor.get_model_stats("stats_model").await;
        assert!(stats.is_some(), "Statistics should be available");

        let stats = stats.unwrap();
        assert_eq!(stats.total_samples, 100);
        assert!((stats.avg_accuracy - 0.75).abs() < 0.01, "Average accuracy should be ~75%");

        // Check latency percentiles
        assert!(stats.p95_latency_us > 900.0, "P95 latency should be near 950");
        assert!(stats.p99_latency_us > 1000.0, "P99 latency should be near 1080");
        assert_eq!(stats.max_latency_us, 1090, "Max latency should be 1090");
    }

    #[tokio::test]
    async fn test_performance_trend_detection() {
        let monitor = create_test_monitor().await;

        // Record 30 samples with improving accuracy
        for i in 0..30 {
            let is_correct = i >= 10; // First 10 wrong, next 20 correct = improving
            let sample = create_sample_with_accuracy("trend_model", is_correct);
            monitor.record_sample(sample).await;
        }

        let stats = monitor.get_model_stats("trend_model").await.unwrap();
        assert_eq!(stats.trend, PerformanceTrend::Improving, "Should detect improving trend");
    }

    // ==================================================================================
    // Test Suite 2: MLFallbackManager Integration
    // ==================================================================================

    #[tokio::test]
    async fn test_model_registration_and_priority() {
        let manager = create_test_fallback_manager().await;

        // Register models with different priorities
        manager.register_model("model_high".to_string(), 100).await;
        manager.register_model("model_medium".to_string(), 50).await;
        manager.register_model("model_low".to_string(), 10).await;

        // Best available should be highest priority
        let best = manager.get_best_available_model().await;
        assert_eq!(best, Some("model_high".to_string()));
    }

    #[tokio::test]
    async fn test_circuit_breaker_state_transitions() {
        let config = create_fallback_config();
        let manager = create_fallback_manager_with_config(config.clone()).await;

        manager.register_model("cb_model".to_string(), 100).await;

        // Record failures to trigger circuit breaker
        for _ in 0..config.circuit_breaker_failure_threshold {
            manager.record_prediction_result("cb_model", false, 100, None).await;
        }

        // Check model status
        let status = manager.get_model_status("cb_model").await;
        assert!(status.is_some());

        let status = status.unwrap();
        assert_eq!(status.health, ModelHealth::Failed);
        assert!(status.consecutive_failures >= config.max_consecutive_failures);
    }

    #[tokio::test]
    async fn test_automatic_failover_on_failures() {
        let manager = create_test_fallback_manager().await;
        let mut event_receiver = manager.subscribe_failover_events();

        // Register primary and backup models
        manager.register_model("primary".to_string(), 100).await;
        manager.register_model("backup".to_string(), 50).await;

        // Cause primary to fail
        for _ in 0..6 {
            manager.record_prediction_result("primary", false, 100, None).await;
        }

        // Wait for failover event
        let event_result = tokio::time::timeout(Duration::from_millis(100), event_receiver.recv()).await;
        assert!(event_result.is_ok(), "Failover event should be broadcast");

        let event = event_result.unwrap().unwrap();
        assert_eq!(event.event_type, FailoverEventType::ModelFailure);
        assert_eq!(event.failed_model, Some("primary".to_string()));
    }

    #[tokio::test]
    async fn test_best_available_model_selection() {
        let manager = create_test_fallback_manager().await;

        // Register models
        manager.register_model("priority_1".to_string(), 100).await;
        manager.register_model("priority_2".to_string(), 80).await;
        manager.register_model("priority_3".to_string(), 60).await;

        // All healthy - should pick highest priority
        let best = manager.get_best_available_model().await;
        assert_eq!(best, Some("priority_1".to_string()));

        // Fail highest priority
        for _ in 0..6 {
            manager.record_prediction_result("priority_1", false, 100, None).await;
        }

        // Should fall back to second priority
        let best = manager.get_best_available_model().await;
        assert_eq!(best, Some("priority_2".to_string()));
    }

    #[tokio::test]
    async fn test_ensemble_prediction_fallback() {
        let manager = create_test_fallback_manager().await;

        // Register multiple models
        manager.register_model("ensemble_1".to_string(), 100).await;
        manager.register_model("ensemble_2".to_string(), 90).await;
        manager.register_model("ensemble_3".to_string(), 80).await;

        // Get ensemble
        let ensemble = manager.get_ensemble_models(3).await;
        assert_eq!(ensemble.len(), 3);
        assert!(ensemble.contains(&"ensemble_1".to_string()));
        assert!(ensemble.contains(&"ensemble_2".to_string()));
        assert!(ensemble.contains(&"ensemble_3".to_string()));
    }

    #[tokio::test]
    async fn test_rule_based_final_fallback() {
        let manager = create_test_fallback_manager().await;

        // No models registered - should fall back to rule-based
        let features = vec![0.05, 1000.0]; // momentum, volume
        let prediction = manager.predict_with_fallback(&features, None).await;

        assert_eq!(prediction.strategy_used, FallbackStrategy::RuleBasedFallback);
        assert_eq!(prediction.models_used, vec!["rule_based".to_string()]);
        assert!(prediction.fallback_triggered);
        assert!(prediction.confidence <= 0.6);
    }

    #[tokio::test]
    async fn test_manual_model_switching() {
        let manager = create_test_fallback_manager().await;

        manager.register_model("model_a".to_string(), 100).await;
        manager.register_model("model_b".to_string(), 50).await;

        // Switch to model_b
        let result = manager.switch_primary_model("model_b".to_string()).await;
        assert!(result.is_ok());

        // Verify event was broadcast
        let events = manager.get_recent_failover_events(1).await;
        assert_eq!(events.len(), 1);
        assert_eq!(events[0].event_type, FailoverEventType::ManualSwitching);
    }

    #[tokio::test]
    async fn test_failover_event_broadcasting() {
        let manager = create_test_fallback_manager().await;
        let mut event_receiver = manager.subscribe_failover_events();

        manager.register_model("event_test".to_string(), 100).await;

        // Trigger failover by causing failures
        for _ in 0..6 {
            manager.record_prediction_result("event_test", false, 100, None).await;
        }

        // Receive event
        let event = tokio::time::timeout(Duration::from_millis(100), event_receiver.recv())
            .await
            .expect("Event should be received")
            .expect("Event should be valid");

        assert_eq!(event.event_type, FailoverEventType::ModelFailure);
        assert_eq!(event.failed_model, Some("event_test".to_string()));
    }

    // ==================================================================================
    // Test Suite 3: Performance Overhead Measurement
    // ==================================================================================

    #[tokio::test]
    async fn test_metric_recording_overhead_under_10us() {
        let iterations = 1000;
        let mut total_overhead_ns = 0u128;

        let monitor = create_test_monitor().await;

        for _i in 0..iterations {
            let sample = create_sample("perf_test", 100, true);

            let start = Instant::now();
            monitor.record_sample(sample).await;
            let elapsed = start.elapsed();

            total_overhead_ns += elapsed.as_nanos();
        }

        let avg_overhead_ns = total_overhead_ns / iterations;
        let avg_overhead_us = avg_overhead_ns as f64 / 1000.0;

        println!("Average metric recording overhead: {:.2}μs ({} ns)", avg_overhead_us, avg_overhead_ns);

        // Wave 67 claimed <10μs overhead
        assert!(avg_overhead_us < 10.0,
            "Metric recording overhead {:.2}μs exceeds 10μs target", avg_overhead_us);
    }

    #[tokio::test]
    async fn test_alert_broadcast_latency() {
        let monitor = create_test_monitor().await;
        let mut receiver = monitor.subscribe_alerts();

        // Configure for immediate alert
        let mut config = AlertConfig::default();
        config.latency_threshold_us = 1;
        monitor.update_config(config).await;

        let sample = create_sample_with_latency("latency_test", 1000);

        let start = Instant::now();
        monitor.record_sample(sample).await;

        let alert = tokio::time::timeout(Duration::from_millis(10), receiver.recv())
            .await
            .expect("Alert should be received quickly")
            .expect("Alert should be valid");

        let broadcast_latency = start.elapsed();

        println!("Alert broadcast latency: {:?}", broadcast_latency);

        // Should be very fast (< 1ms for local broadcast)
        assert!(broadcast_latency < Duration::from_millis(1),
            "Alert broadcast took {:?}, expected <1ms", broadcast_latency);
    }

    #[tokio::test]
    async fn test_failover_decision_latency() {
        let manager = create_test_fallback_manager().await;

        manager.register_model("model_1".to_string(), 100).await;
        manager.register_model("model_2".to_string(), 50).await;

        let features = vec![0.1, 2000.0];

        // Measure prediction with fallback latency
        let start = Instant::now();
        let _prediction = manager.predict_with_fallback(&features, Some("model_1".to_string())).await;
        let decision_latency = start.elapsed();

        println!("Failover decision latency: {:?}", decision_latency);

        // Should be sub-millisecond for local operations
        assert!(decision_latency < Duration::from_millis(1),
            "Failover decision took {:?}, expected <1ms", decision_latency);
    }

    // ==================================================================================
    // Test Suite 4: Cross-Component Integration
    // ==================================================================================

    #[tokio::test]
    async fn test_end_to_end_prediction_with_monitoring() {
        let monitor = create_test_monitor().await;
        let manager = create_test_fallback_manager().await;

        // Register models
        manager.register_model("integrated_model".to_string(), 100).await;

        // Make prediction
        let features = vec![0.05, 1500.0];
        let prediction = manager.predict_with_fallback(&features, Some("integrated_model".to_string())).await;

        // Record performance sample based on prediction
        let sample = ModelPerformanceSample {
            model_id: prediction.models_used[0].clone(),
            timestamp: SystemTime::now(),
            accuracy: prediction.confidence,
            latency_us: prediction.latency_us,
            confidence: prediction.confidence,
            memory_usage_mb: 128.0,
            cpu_utilization: 25.0,
            prediction_correct: Some(true),
            prediction_error: None,
            market_regime: Some("normal".to_string()),
        };

        monitor.record_sample(sample).await;

        // Verify stats were updated
        let stats = monitor.get_model_stats("integrated_model").await;
        assert!(stats.is_some());
    }

    #[tokio::test]
    async fn test_alert_triggers_failover() {
        let monitor = create_test_monitor().await;
        let manager = create_test_fallback_manager().await;

        let mut alert_receiver = monitor.subscribe_alerts();
        let mut failover_receiver = manager.subscribe_failover_events();

        // Register models
        manager.register_model("failing_model".to_string(), 100).await;
        manager.register_model("backup_model".to_string(), 50).await;

        // Simulate failures that trigger both alerts and failover
        for _ in 0..6 {
            let sample = create_sample_with_latency("failing_model", 5000);
            monitor.record_sample(sample).await;
            manager.record_prediction_result("failing_model", false, 5000, Some(0.3)).await;
        }

        // Should receive both alert and failover event
        let alert_result = tokio::time::timeout(Duration::from_millis(100), alert_receiver.recv()).await;
        let failover_result = tokio::time::timeout(Duration::from_millis(100), failover_receiver.recv()).await;

        assert!(alert_result.is_ok(), "Alert should be triggered");
        assert!(failover_result.is_ok(), "Failover should be triggered");
    }

    // ==================================================================================
    // Helper Functions
    // ==================================================================================

    async fn create_test_monitor() -> MLPerformanceMonitor {
        MLPerformanceMonitor::new()
    }

    async fn create_monitor_with_config(config: AlertConfig) -> MLPerformanceMonitor {
        MLPerformanceMonitor::with_config(config)
    }

    async fn create_test_fallback_manager() -> MLFallbackManager {
        MLFallbackManager::new()
    }

    async fn create_fallback_manager_with_config(config: FallbackConfig) -> MLFallbackManager {
        let manager = MLFallbackManager::new();
        manager.update_config(config).await;
        manager
    }

    fn create_fallback_config() -> FallbackConfig {
        FallbackConfig::default()
    }

    fn create_high_latency_sample(model_id: &str, latency_us: u64) -> ModelPerformanceSample {
        ModelPerformanceSample {
            model_id: model_id.to_string(),
            timestamp: SystemTime::now(),
            accuracy: 0.85,
            latency_us,
            confidence: 0.9,
            memory_usage_mb: 100.0,
            cpu_utilization: 25.0,
            prediction_correct: Some(true),
            prediction_error: Some(0.1),
            market_regime: Some("normal".to_string()),
        }
    }

    fn create_sample_with_latency(model_id: &str, latency_us: u64) -> ModelPerformanceSample {
        ModelPerformanceSample {
            model_id: model_id.to_string(),
            timestamp: SystemTime::now(),
            accuracy: 0.85,
            latency_us,
            confidence: 0.9,
            memory_usage_mb: 100.0,
            cpu_utilization: 25.0,
            prediction_correct: Some(true),
            prediction_error: None,
            market_regime: Some("normal".to_string()),
        }
    }

    fn create_sample_with_accuracy(model_id: &str, is_correct: bool) -> ModelPerformanceSample {
        ModelPerformanceSample {
            model_id: model_id.to_string(),
            timestamp: SystemTime::now(),
            accuracy: if is_correct { 0.95 } else { 0.3 },
            latency_us: 500,
            confidence: 0.9,
            memory_usage_mb: 100.0,
            cpu_utilization: 25.0,
            prediction_correct: Some(is_correct),
            prediction_error: if is_correct { Some(0.05) } else { Some(0.7) },
            market_regime: Some("normal".to_string()),
        }
    }

    fn create_sample_with_memory(model_id: &str, memory_mb: f64) -> ModelPerformanceSample {
        ModelPerformanceSample {
            model_id: model_id.to_string(),
            timestamp: SystemTime::now(),
            accuracy: 0.85,
            latency_us: 500,
            confidence: 0.9,
            memory_usage_mb: memory_mb,
            cpu_utilization: 25.0,
            prediction_correct: Some(true),
            prediction_error: None,
            market_regime: Some("normal".to_string()),
        }
    }

    fn create_sample(model_id: &str, latency_us: u64, is_correct: bool) -> ModelPerformanceSample {
        ModelPerformanceSample {
            model_id: model_id.to_string(),
            timestamp: SystemTime::now(),
            accuracy: if is_correct { 0.9 } else { 0.4 },
            latency_us,
            confidence: 0.85,
            memory_usage_mb: 128.0,
            cpu_utilization: 30.0,
            prediction_correct: Some(is_correct),
            prediction_error: if is_correct { Some(0.1) } else { Some(0.6) },
            market_regime: Some("normal".to_string()),
        }
    }

    // Import types from trading_service
    // These would normally be imported from the actual modules
    // For now, we'll define stub types for compilation

    use serde::{Deserialize, Serialize};

    #[derive(Debug, Clone, Serialize, Deserialize)]
    pub struct ModelPerformanceSample {
        pub model_id: String,
        pub timestamp: SystemTime,
        pub accuracy: f64,
        pub latency_us: u64,
        pub confidence: f64,
        pub memory_usage_mb: f64,
        pub cpu_utilization: f64,
        pub prediction_correct: Option<bool>,
        pub prediction_error: Option<f64>,
        pub market_regime: Option<String>,
    }

    #[derive(Debug, Clone, Serialize, Deserialize)]
    pub struct AlertConfig {
        pub enable_latency_alerts: bool,
        pub latency_threshold_us: u64,
        pub enable_accuracy_alerts: bool,
        pub accuracy_threshold: f64,
        pub enable_memory_alerts: bool,
        pub memory_threshold_mb: f64,
        pub alert_cooldown_seconds: u64,
        pub enable_drift_detection: bool,
        pub drift_window_size: usize,
        pub drift_threshold_percent: f64,
    }

    impl Default for AlertConfig {
        fn default() -> Self {
            Self {
                enable_latency_alerts: true,
                latency_threshold_us: 1000,
                enable_accuracy_alerts: true,
                accuracy_threshold: 0.65,
                enable_memory_alerts: true,
                memory_threshold_mb: 512.0,
                alert_cooldown_seconds: 300,
                enable_drift_detection: true,
                drift_window_size: 100,
                drift_threshold_percent: 10.0,
            }
        }
    }

    #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
    pub enum AlertType {
        HighLatency,
        LowAccuracy,
        HighMemoryUsage,
        ModelDrift,
        ModelFailure,
        PredictionAnomaly,
    }

    #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
    pub enum AlertSeverity {
        Info,
        Warning,
        Critical,
        Emergency,
    }

    #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq)]
    pub enum PerformanceTrend {
        Improving,
        Stable,
        Degrading,
        Unknown,
    }

    #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
    pub enum ModelHealth {
        Healthy,
        Degraded,
        Unhealthy,
        Failed,
        Offline,
    }

    #[derive(Debug, Clone, Serialize, Deserialize)]
    pub struct FallbackConfig {
        pub min_healthy_models: usize,
        pub max_consecutive_failures: u32,
        pub min_success_rate: f64,
        pub max_latency_us: u64,
        pub min_accuracy: f64,
        pub health_check_interval_seconds: u64,
        pub circuit_breaker_failure_threshold: u32,
        pub circuit_breaker_timeout_seconds: u64,
        pub enable_auto_switching: bool,
        pub fallback_timeout_ms: u64,
    }

    impl Default for FallbackConfig {
        fn default() -> Self {
            Self {
                min_healthy_models: 1,
                max_consecutive_failures: 5,
                min_success_rate: 0.7,
                max_latency_us: 5000,
                min_accuracy: 0.6,
                health_check_interval_seconds: 30,
                circuit_breaker_failure_threshold: 10,
                circuit_breaker_timeout_seconds: 60,
                enable_auto_switching: true,
                fallback_timeout_ms: 100,
            }
        }
    }

    #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
    pub enum FallbackStrategy {
        PriorityBased,
        PerformanceBased,
        EnsembleBased,
        RuleBasedFallback,
        NeutralFallback,
    }

    #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
    pub enum FailoverEventType {
        ModelFailure,
        ModelDegraded,
        CircuitBreakerOpen,
        AutoSwitching,
        ManualSwitching,
        Recovery,
    }

    // Mock implementations for testing
    pub struct MLPerformanceMonitor {
        // Implementation would be in trading_service
    }

    impl MLPerformanceMonitor {
        pub fn new() -> Self {
            Self {}
        }

        pub fn with_config(_config: AlertConfig) -> Self {
            Self {}
        }

        pub async fn record_sample(&self, _sample: ModelPerformanceSample) {}

        pub fn subscribe_alerts(&self) -> tokio::sync::broadcast::Receiver<PerformanceAlert> {
            let (tx, rx) = tokio::sync::broadcast::channel(100);
            rx
        }

        pub async fn get_model_stats(&self, _model_id: &str) -> Option<ModelPerformanceStats> {
            Some(ModelPerformanceStats::default())
        }

        pub async fn update_config(&self, _config: AlertConfig) {}
    }

    pub struct MLFallbackManager {
        // Implementation would be in trading_service
    }

    impl MLFallbackManager {
        pub fn new() -> Self {
            Self {}
        }

        pub async fn register_model(&self, _model_id: String, _priority: i32) {}

        pub async fn record_prediction_result(
            &self,
            _model_id: &str,
            _success: bool,
            _latency_us: u64,
            _accuracy: Option<f64>,
        ) {
        }

        pub async fn get_best_available_model(&self) -> Option<String> {
            Some("test_model".to_string())
        }

        pub async fn get_ensemble_models(&self, _max: usize) -> Vec<String> {
            vec![]
        }

        pub async fn predict_with_fallback(
            &self,
            _features: &[f64],
            _preferred: Option<String>,
        ) -> FallbackPrediction {
            FallbackPrediction {
                prediction_value: 0.5,
                confidence: 0.8,
                models_used: vec!["test".to_string()],
                strategy_used: FallbackStrategy::PriorityBased,
                fallback_triggered: false,
                latency_us: 100,
                warnings: vec![],
            }
        }

        pub fn subscribe_failover_events(&self) -> tokio::sync::broadcast::Receiver<FailoverEvent> {
            let (tx, rx) = tokio::sync::broadcast::channel(100);
            rx
        }

        pub async fn get_model_status(&self, _model_id: &str) -> Option<ModelStatus> {
            None
        }

        pub async fn get_recent_failover_events(&self, _limit: usize) -> Vec<FailoverEvent> {
            vec![]
        }

        pub async fn switch_primary_model(&self, _model_id: String) -> Result<(), String> {
            Ok(())
        }

        pub async fn update_config(&self, _config: FallbackConfig) {}
    }

    #[derive(Debug, Clone, Serialize, Deserialize)]
    pub struct PerformanceAlert {
        pub alert_id: String,
        pub timestamp: SystemTime,
        pub severity: AlertSeverity,
        pub alert_type: AlertType,
        pub model_id: String,
        pub message: String,
        pub current_value: f64,
        pub threshold: f64,
        pub suggested_action: String,
    }

    #[derive(Debug, Clone, Serialize, Deserialize)]
    pub struct ModelPerformanceStats {
        pub model_id: String,
        pub total_samples: u64,
        pub avg_accuracy: f64,
        pub p95_latency_us: f64,
        pub p99_latency_us: f64,
        pub max_latency_us: u64,
        pub avg_memory_mb: f64,
        pub peak_memory_mb: f64,
        pub avg_cpu_utilization: f64,
        pub error_rate: f64,
        pub trend: PerformanceTrend,
        pub last_updated: SystemTime,
    }

    impl Default for ModelPerformanceStats {
        fn default() -> Self {
            Self {
                model_id: String::new(),
                total_samples: 0,
                avg_accuracy: 0.0,
                p95_latency_us: 0.0,
                p99_latency_us: 0.0,
                max_latency_us: 0,
                avg_memory_mb: 0.0,
                peak_memory_mb: 0.0,
                avg_cpu_utilization: 0.0,
                error_rate: 0.0,
                trend: PerformanceTrend::Unknown,
                last_updated: SystemTime::now(),
            }
        }
    }

    #[derive(Debug, Clone, Serialize, Deserialize)]
    pub struct FallbackPrediction {
        pub prediction_value: f64,
        pub confidence: f64,
        pub models_used: Vec<String>,
        pub strategy_used: FallbackStrategy,
        pub fallback_triggered: bool,
        pub latency_us: u64,
        pub warnings: Vec<String>,
    }

    #[derive(Debug, Clone, Serialize, Deserialize)]
    pub struct FailoverEvent {
        pub timestamp: SystemTime,
        pub event_type: FailoverEventType,
        pub failed_model: Option<String>,
        pub fallback_model: Option<String>,
        pub strategy: FallbackStrategy,
        pub message: String,
        pub impact: FailoverImpact,
    }

    #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
    pub enum FailoverImpact {
        None,
        Low,
        Medium,
        High,
        Critical,
    }

    #[derive(Debug, Clone, Serialize, Deserialize)]
    pub struct ModelStatus {
        pub model_id: String,
        pub health: ModelHealth,
        pub last_success: Option<SystemTime>,
        pub consecutive_failures: u32,
        pub total_predictions: u64,
        pub success_rate: f64,
        pub avg_latency_us: f64,
        pub accuracy_score: f64,
        pub priority: i32,
        pub enabled: bool,
        pub last_health_check: SystemTime,
        pub circuit_breaker_state: CircuitBreakerState,
    }

    #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
    pub enum CircuitBreakerState {
        Closed,
        Open,
        HalfOpen,
    }
}