foxhunt/services/ml_training_service/tests/monitoring_tests.rs

//! Comprehensive Monitoring Tests for ML Training Service
//!
//! TDD approach: Tests written first, implementation follows to make tests pass.

use chrono::{DateTime, Duration, Utc};
use serde_json::Value;

#[cfg(test)]
mod alert_evaluation_tests {
    use super::*;

    #[tokio::test]
    async fn test_gpu_memory_high_alert_triggers() {
        // Arrange: GPU memory >90%
        let gpu_metrics = GpuMetrics {
            gpu_id: "0".to_string(),
            memory_used_bytes: 95.0 * 1e9,  // 95% of 100GB
            memory_total_bytes: 100.0 * 1e9,
            utilization_percent: 85.0,
            temperature_celsius: 75.0,
            timestamp: Utc::now(),
        };

        let monitor = MonitoringSystem::new(MonitoringConfig::default()).await.unwrap();

        // Act
        let alerts = monitor.evaluate_gpu_alerts(&gpu_metrics).await.unwrap();

        // Assert
        assert!(alerts.iter().any(|a| a.name == "GPUMemoryUsageHigh"));
        let alert = alerts.iter().find(|a| a.name == "GPUMemoryUsageHigh").unwrap();
        assert_eq!(alert.severity, AlertSeverity::Warning);
        assert_eq!(alert.component, "ml");
        assert!(alert.description.contains("95"));
    }

    #[tokio::test]
    async fn test_gpu_memory_exhausted_alert_critical() {
        // Arrange: GPU memory >95% (CRITICAL)
        let gpu_metrics = GpuMetrics {
            gpu_id: "0".to_string(),
            memory_used_bytes: 97.0 * 1e9,  // 97% of 100GB
            memory_total_bytes: 100.0 * 1e9,
            utilization_percent: 98.0,
            temperature_celsius: 80.0,
            timestamp: Utc::now(),
        };

        let monitor = MonitoringSystem::new(MonitoringConfig::default()).await.unwrap();

        // Act
        let alerts = monitor.evaluate_gpu_alerts(&gpu_metrics).await.unwrap();

        // Assert
        assert!(alerts.iter().any(|a| a.name == "GPUMemoryExhausted"));
        let alert = alerts.iter().find(|a| a.name == "GPUMemoryExhausted").unwrap();
        assert_eq!(alert.severity, AlertSeverity::Critical);
        assert!(alert.action.is_some());
        assert!(alert.action.as_ref().unwrap().contains("Reduce batch size"));
    }

    #[tokio::test]
    async fn test_training_job_failure_alert() {
        // Arrange: Job failed with error
        let job_event = TrainingJobEvent {
            job_id: "job-123".to_string(),
            model_type: "DQN".to_string(),
            status: JobStatus::Failed,
            error_message: Some("NaN values detected in loss".to_string()),
            timestamp: Utc::now(),
        };

        let monitor = MonitoringSystem::new(MonitoringConfig::default()).await.unwrap();

        // Act
        let alerts = monitor.evaluate_job_alerts(&job_event).await.unwrap();

        // Assert
        assert!(alerts.iter().any(|a| a.name == "TrainingJobFailed"));
        let alert = alerts.iter().find(|a| a.name == "TrainingJobFailed").unwrap();
        assert_eq!(alert.severity, AlertSeverity::High);
        assert!(alert.description.contains("NaN values"));
    }

    #[tokio::test]
    async fn test_s3_storage_high_alert() {
        // Arrange: S3 storage >1TB
        let storage_metrics = StorageMetrics {
            total_bytes: 1.2e12,  // 1.2TB
            used_bytes: 1.1e12,   // 1.1TB used
            object_count: 5000,
            timestamp: Utc::now(),
        };

        let monitor = MonitoringSystem::new(MonitoringConfig::default()).await.unwrap();

        // Act
        let alerts = monitor.evaluate_storage_alerts(&storage_metrics).await.unwrap();

        // Assert
        assert!(alerts.iter().any(|a| a.name == "S3StorageUsageHigh"));
        let alert = alerts.iter().find(|a| a.name == "S3StorageUsageHigh").unwrap();
        assert_eq!(alert.severity, AlertSeverity::Warning);
        assert!(alert.description.contains("1TB"));
    }

    #[tokio::test]
    async fn test_data_drift_alert() {
        // Arrange: Feature distribution shift detected
        let drift_metrics = DataDriftMetrics {
            feature_name: "rsi_14".to_string(),
            drift_score: 0.22,  // Above 0.15 threshold
            distribution_distance: 0.25,
            timestamp: Utc::now(),
        };

        let monitor = MonitoringSystem::new(MonitoringConfig::default()).await.unwrap();

        // Act
        let alerts = monitor.evaluate_drift_alerts(&drift_metrics).await.unwrap();

        // Assert
        assert!(alerts.iter().any(|a| a.name == "DataDriftDetected"));
        let alert = alerts.iter().find(|a| a.name == "DataDriftDetected").unwrap();
        assert_eq!(alert.severity, AlertSeverity::Warning);
        assert!(alert.description.contains("rsi_14"));
        assert!(alert.description.contains("0.22"));
    }
}

#[cfg(test)]
mod notification_integration_tests {
    use super::*;

    #[tokio::test]
    async fn test_slack_webhook_success() {
        // Arrange: Mock Slack webhook
        let config = NotificationConfig {
            slack_webhook_url: Some("https://hooks.slack.com/services/mock".to_string()),
            pagerduty_integration_key: None,
            enabled: true,
        };
        let notifier = NotificationService::new(config).await.unwrap();

        let alert = Alert {
            name: "GPUMemoryHigh".to_string(),
            severity: AlertSeverity::Warning,
            component: "ml".to_string(),
            summary: "GPU memory usage high".to_string(),
            description: "GPU 0 memory 92%".to_string(),
            impact: Some("Risk of OOM".to_string()),
            action: Some("Reduce batch size".to_string()),
            timestamp: Utc::now(),
            labels: vec![("gpu_id".to_string(), "0".to_string())],
            runbook_url: None,
        };

        // Act
        let result = notifier.send_slack_notification(&alert).await;

        // Assert: Should succeed with mock webhook
        assert!(result.is_ok());
    }

    #[tokio::test]
    async fn test_pagerduty_webhook_success() {
        // Arrange: Mock PagerDuty integration
        let config = NotificationConfig {
            slack_webhook_url: None,
            pagerduty_integration_key: Some("test-key-123".to_string()),
            enabled: true,
        };
        let notifier = NotificationService::new(config).await.unwrap();

        let alert = Alert {
            name: "TrainingJobCrashed".to_string(),
            severity: AlertSeverity::Critical,
            component: "ml".to_string(),
            summary: "Training job crashed".to_string(),
            description: "Job job-456 crashed with OOM".to_string(),
            impact: Some("Model training lost".to_string()),
            action: Some("Restart with smaller batch size".to_string()),
            timestamp: Utc::now(),
            labels: vec![("job_id".to_string(), "job-456".to_string())],
            runbook_url: Some("https://docs.foxhunt.io/runbooks/oom".to_string()),
        };

        // Act
        let result = notifier.send_pagerduty_notification(&alert).await;

        // Assert: Should succeed with mock key
        assert!(result.is_ok());
    }

    #[tokio::test]
    async fn test_notification_disabled() {
        // Arrange: Notifications disabled
        let config = NotificationConfig {
            slack_webhook_url: Some("https://hooks.slack.com/services/mock".to_string()),
            pagerduty_integration_key: Some("test-key".to_string()),
            enabled: false,
        };
        let notifier = NotificationService::new(config).await.unwrap();

        let alert = Alert {
            name: "TestAlert".to_string(),
            severity: AlertSeverity::Info,
            component: "ml".to_string(),
            summary: "Test".to_string(),
            description: "Test alert".to_string(),
            impact: None,
            action: None,
            timestamp: Utc::now(),
            labels: vec![],
            runbook_url: None,
        };

        // Act
        let result = notifier.send_slack_notification(&alert).await;

        // Assert: Should skip silently
        assert!(result.is_ok());
    }

    #[tokio::test]
    async fn test_alert_deduplication() {
        // Arrange: Same alert sent twice within 5 minutes
        let config = NotificationConfig {
            slack_webhook_url: Some("https://hooks.slack.com/services/mock".to_string()),
            pagerduty_integration_key: None,
            enabled: true,
        };
        let notifier = NotificationService::new(config).await.unwrap();

        let alert = Alert {
            name: "GPUMemoryHigh".to_string(),
            severity: AlertSeverity::Warning,
            component: "ml".to_string(),
            summary: "GPU memory high".to_string(),
            description: "GPU 0 memory 92%".to_string(),
            impact: None,
            action: None,
            timestamp: Utc::now(),
            labels: vec![("gpu_id".to_string(), "0".to_string())],
            runbook_url: None,
        };

        // Act: Send same alert twice
        let result1 = notifier.send_slack_notification(&alert).await;
        let result2 = notifier.send_slack_notification(&alert).await;

        // Assert: First should succeed, second should be deduplicated
        assert!(result1.is_ok());
        assert!(result2.is_ok());
        // Check deduplication count
        let stats = notifier.get_statistics().await.unwrap();
        assert_eq!(stats.deduplicated_alerts, 1);
    }
}

#[cfg(test)]
mod cost_tracking_tests {
    use super::*;

    #[tokio::test]
    async fn test_s3_storage_cost_calculation() {
        // Arrange: 500GB S3 storage
        let storage_bytes: f64 = 500.0 * 1e9;  // 500GB
        let cost_tracker = CostTracker::new(CostConfig::default()).await.unwrap();

        // Act
        let monthly_cost = cost_tracker.calculate_s3_cost(storage_bytes).await.unwrap();

        // Assert: S3 Standard costs ~$0.023/GB/month
        // 500GB * $0.023 = $11.50/month
        assert!(monthly_cost > 10.0 && monthly_cost < 15.0);
    }

    #[tokio::test]
    async fn test_gpu_hours_cost_calculation() {
        // Arrange: 100 GPU hours on RTX 3050 Ti
        let gpu_hours: f64 = 100.0;
        let gpu_type = "RTX_3050_Ti".to_string();
        let cost_tracker = CostTracker::new(CostConfig::default()).await.unwrap();

        // Act
        let total_cost = cost_tracker.calculate_gpu_cost(gpu_hours, &gpu_type).await.unwrap();

        // Assert: Local GPU cost assumed $0 (already owned)
        // Cloud GPU would be ~$1-2/hour
        assert_eq!(total_cost, 0.0);  // Local GPU
    }

    #[tokio::test]
    async fn test_cloud_gpu_cost_calculation() {
        // Arrange: 50 hours on A100 GPU
        let gpu_hours: f64 = 50.0;
        let gpu_type = "A100".to_string();
        let cost_tracker = CostTracker::new(CostConfig::default()).await.unwrap();

        // Act
        let total_cost = cost_tracker.calculate_gpu_cost(gpu_hours, &gpu_type).await.unwrap();

        // Assert: A100 costs ~$2.50/hour on most cloud providers
        // 50 hours * $2.50 = $125
        assert!(total_cost > 100.0 && total_cost < 150.0);
    }

    #[tokio::test]
    async fn test_cost_alert_threshold() {
        // Arrange: Monthly cost exceeds budget
        let cost_tracker = CostTracker::new(CostConfig {
            monthly_budget: 500.0,  // $500/month budget
            alert_threshold_percent: 80.0,  // Alert at 80%
            ..Default::default()
        }).await.unwrap();

        // Record costs
        cost_tracker.record_s3_cost(200.0).await.unwrap();
        cost_tracker.record_gpu_cost(250.0).await.unwrap();

        // Act
        let alerts = cost_tracker.check_cost_alerts().await.unwrap();

        // Assert: Should trigger alert (450/500 = 90% of budget)
        assert!(alerts.iter().any(|a| a.name == "MonthlyCostHighAlert"));
        let alert = alerts.iter().find(|a| a.name == "MonthlyCostHighAlert").unwrap();
        assert_eq!(alert.severity, AlertSeverity::Warning);
        assert!(alert.description.contains("90%"));
    }

    #[tokio::test]
    async fn test_cost_projection() {
        // Arrange: Cost tracker with historical data
        let cost_tracker = CostTracker::new(CostConfig::default()).await.unwrap();

        // Record costs over multiple days
        for day in 1..=10 {
            cost_tracker.record_daily_cost(day, 50.0).await.unwrap();
        }

        // Act
        let projected_monthly_cost = cost_tracker.project_monthly_cost().await.unwrap();

        // Assert: $50/day * 30 days = $1500/month
        assert!(projected_monthly_cost > 1400.0 && projected_monthly_cost < 1600.0);
    }
}

#[cfg(test)]
mod data_drift_detection_tests {
    use super::*;

    #[tokio::test]
    async fn test_feature_distribution_shift() {
        // Arrange: Training data and production data
        let training_data = vec![1.0, 2.0, 3.0, 4.0, 5.0];
        let production_data = vec![10.0, 20.0, 30.0, 40.0, 50.0];  // Significantly different

        let drift_detector = DataDriftDetector::new(DriftConfig::default()).await.unwrap();

        // Act
        let drift_score = drift_detector
            .calculate_drift("rsi_14", &training_data, &production_data)
            .await
            .unwrap();

        // Assert: Should detect significant drift
        assert!(drift_score > 0.5);  // High drift score
    }

    #[tokio::test]
    async fn test_no_drift_detected() {
        // Arrange: Similar distributions
        let training_data = vec![1.0, 2.0, 3.0, 4.0, 5.0];
        let production_data = vec![1.1, 2.0, 2.9, 4.1, 5.0];  // Very similar

        let drift_detector = DataDriftDetector::new(DriftConfig::default()).await.unwrap();

        // Act
        let drift_score = drift_detector
            .calculate_drift("rsi_14", &training_data, &production_data)
            .await
            .unwrap();

        // Assert: Should detect minimal drift
        assert!(drift_score < 0.1);  // Low drift score
    }

    #[tokio::test]
    async fn test_kolmogorov_smirnov_test() {
        // Arrange: Two distributions to compare
        let dist1 = vec![1.0, 2.0, 3.0, 4.0, 5.0];
        let dist2 = vec![5.0, 6.0, 7.0, 8.0, 9.0];

        let drift_detector = DataDriftDetector::new(DriftConfig::default()).await.unwrap();

        // Act
        let ks_statistic = drift_detector
            .ks_test(&dist1, &dist2)
            .await
            .unwrap();

        // Assert: KS statistic should be high (distributions are different)
        assert!(ks_statistic > 0.5);
    }

    #[tokio::test]
    async fn test_drift_alert_generation() {
        // Arrange: Drift detector with threshold
        let drift_detector = DataDriftDetector::new(DriftConfig {
            drift_threshold: 0.15,
            check_interval_minutes: 60,
            ..Default::default()
        }).await.unwrap();

        // Record drift above threshold
        drift_detector.record_drift("macd", 0.25).await.unwrap();

        // Act
        let alerts = drift_detector.check_drift_alerts().await.unwrap();

        // Assert: Should generate drift alert
        assert!(alerts.iter().any(|a| a.name == "DataDriftDetected"));
        let alert = alerts.iter().find(|a| a.name == "DataDriftDetected").unwrap();
        assert!(alert.description.contains("macd"));
        assert!(alert.description.contains("0.25"));
    }
}

// ============================================================================
// Supporting Types (to be implemented in monitoring.rs)
// ============================================================================

#[derive(Debug, Clone)]
pub struct MonitoringSystem {
    config: MonitoringConfig,
    alert_manager: AlertManager,
    cost_tracker: CostTracker,
    drift_detector: DataDriftDetector,
}

#[derive(Debug, Clone)]
pub struct MonitoringConfig {
    pub alert_evaluation_interval_secs: u64,
    pub enable_notifications: bool,
    pub enable_cost_tracking: bool,
    pub enable_drift_detection: bool,
}

impl Default for MonitoringConfig {
    fn default() -> Self {
        Self {
            alert_evaluation_interval_secs: 30,
            enable_notifications: true,
            enable_cost_tracking: true,
            enable_drift_detection: true,
        }
    }
}

#[derive(Debug, Clone)]
pub struct GpuMetrics {
    pub gpu_id: String,
    pub memory_used_bytes: f64,
    pub memory_total_bytes: f64,
    pub utilization_percent: f64,
    pub temperature_celsius: f64,
    pub timestamp: DateTime<Utc>,
}

#[derive(Debug, Clone)]
pub struct StorageMetrics {
    pub total_bytes: f64,
    pub used_bytes: f64,
    pub object_count: u64,
    pub timestamp: DateTime<Utc>,
}

#[derive(Debug, Clone)]
pub struct DataDriftMetrics {
    pub feature_name: String,
    pub drift_score: f64,
    pub distribution_distance: f64,
    pub timestamp: DateTime<Utc>,
}

#[derive(Debug, Clone)]
pub struct TrainingJobEvent {
    pub job_id: String,
    pub model_type: String,
    pub status: JobStatus,
    pub error_message: Option<String>,
    pub timestamp: DateTime<Utc>,
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum JobStatus {
    Pending,
    Running,
    Completed,
    Failed,
    Stopped,
}

#[derive(Debug, Clone)]
pub struct Alert {
    pub name: String,
    pub severity: AlertSeverity,
    pub component: String,
    pub summary: String,
    pub description: String,
    pub impact: Option<String>,
    pub action: Option<String>,
    pub timestamp: DateTime<Utc>,
    pub labels: Vec<(String, String)>,
    pub runbook_url: Option<String>,
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum AlertSeverity {
    Info,
    Warning,
    High,
    Critical,
}

#[derive(Debug, Clone)]
pub struct NotificationConfig {
    pub slack_webhook_url: Option<String>,
    pub pagerduty_integration_key: Option<String>,
    pub enabled: bool,
}

#[derive(Debug, Clone)]
pub struct NotificationService {
    config: NotificationConfig,
    deduplication_cache: std::sync::Arc<tokio::sync::Mutex<std::collections::HashMap<String, DateTime<Utc>>>>,
    stats: std::sync::Arc<tokio::sync::Mutex<NotificationStats>>,
}

#[derive(Debug, Clone, Default)]
pub struct NotificationStats {
    pub total_sent: u64,
    pub deduplicated_alerts: u64,
    pub failed_notifications: u64,
}

#[derive(Debug, Clone)]
pub struct CostConfig {
    pub s3_cost_per_gb_month: f64,
    pub monthly_budget: f64,
    pub alert_threshold_percent: f64,
}

impl Default for CostConfig {
    fn default() -> Self {
        Self {
            s3_cost_per_gb_month: 0.023,  // AWS S3 Standard
            monthly_budget: 1000.0,
            alert_threshold_percent: 80.0,
        }
    }
}

#[derive(Debug, Clone)]
pub struct CostTracker {
    config: CostConfig,
    daily_costs: std::sync::Arc<tokio::sync::Mutex<std::collections::HashMap<u32, f64>>>,
}

#[derive(Debug, Clone)]
pub struct DriftConfig {
    pub drift_threshold: f64,
    pub check_interval_minutes: u64,
}

impl Default for DriftConfig {
    fn default() -> Self {
        Self {
            drift_threshold: 0.15,
            check_interval_minutes: 60,
        }
    }
}

#[derive(Debug, Clone)]
pub struct DataDriftDetector {
    config: DriftConfig,
    drift_history: std::sync::Arc<tokio::sync::Mutex<std::collections::HashMap<String, Vec<f64>>>>,
}

#[derive(Debug, Clone)]
pub struct AlertManager {
    alerts: std::sync::Arc<tokio::sync::Mutex<Vec<Alert>>>,
}

impl MonitoringSystem {
    pub async fn new(config: MonitoringConfig) -> anyhow::Result<Self> {
        Ok(Self {
            config: config.clone(),
            alert_manager: AlertManager::new().await?,
            cost_tracker: CostTracker::new(CostConfig::default()).await?,
            drift_detector: DataDriftDetector::new(DriftConfig::default()).await?,
        })
    }

    pub async fn evaluate_gpu_alerts(&self, metrics: &GpuMetrics) -> anyhow::Result<Vec<Alert>> {
        unimplemented!("To be implemented")
    }

    pub async fn evaluate_job_alerts(&self, event: &TrainingJobEvent) -> anyhow::Result<Vec<Alert>> {
        unimplemented!("To be implemented")
    }

    pub async fn evaluate_storage_alerts(&self, metrics: &StorageMetrics) -> anyhow::Result<Vec<Alert>> {
        unimplemented!("To be implemented")
    }

    pub async fn evaluate_drift_alerts(&self, metrics: &DataDriftMetrics) -> anyhow::Result<Vec<Alert>> {
        unimplemented!("To be implemented")
    }
}

impl NotificationService {
    pub async fn new(config: NotificationConfig) -> anyhow::Result<Self> {
        Ok(Self {
            config,
            deduplication_cache: std::sync::Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
            stats: std::sync::Arc::new(tokio::sync::Mutex::new(NotificationStats::default())),
        })
    }

    pub async fn send_slack_notification(&self, alert: &Alert) -> anyhow::Result<()> {
        unimplemented!("To be implemented")
    }

    pub async fn send_pagerduty_notification(&self, alert: &Alert) -> anyhow::Result<()> {
        unimplemented!("To be implemented")
    }

    pub async fn get_statistics(&self) -> anyhow::Result<NotificationStats> {
        let stats = self.stats.lock().await;
        Ok(stats.clone())
    }
}

impl CostTracker {
    pub async fn new(config: CostConfig) -> anyhow::Result<Self> {
        Ok(Self {
            config,
            daily_costs: std::sync::Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
        })
    }

    pub async fn calculate_s3_cost(&self, storage_bytes: f64) -> anyhow::Result<f64> {
        unimplemented!("To be implemented")
    }

    pub async fn calculate_gpu_cost(&self, gpu_hours: f64, gpu_type: &str) -> anyhow::Result<f64> {
        unimplemented!("To be implemented")
    }

    pub async fn record_s3_cost(&self, cost: f64) -> anyhow::Result<()> {
        unimplemented!("To be implemented")
    }

    pub async fn record_gpu_cost(&self, cost: f64) -> anyhow::Result<()> {
        unimplemented!("To be implemented")
    }

    pub async fn record_daily_cost(&self, day: u32, cost: f64) -> anyhow::Result<()> {
        unimplemented!("To be implemented")
    }

    pub async fn check_cost_alerts(&self) -> anyhow::Result<Vec<Alert>> {
        unimplemented!("To be implemented")
    }

    pub async fn project_monthly_cost(&self) -> anyhow::Result<f64> {
        unimplemented!("To be implemented")
    }
}

impl DataDriftDetector {
    pub async fn new(config: DriftConfig) -> anyhow::Result<Self> {
        Ok(Self {
            config,
            drift_history: std::sync::Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
        })
    }

    pub async fn calculate_drift(
        &self,
        feature_name: &str,
        training_data: &[f64],
        production_data: &[f64],
    ) -> anyhow::Result<f64> {
        unimplemented!("To be implemented")
    }

    pub async fn ks_test(&self, dist1: &[f64], dist2: &[f64]) -> anyhow::Result<f64> {
        unimplemented!("To be implemented")
    }

    pub async fn record_drift(&self, feature_name: &str, drift_score: f64) -> anyhow::Result<()> {
        unimplemented!("To be implemented")
    }

    pub async fn check_drift_alerts(&self) -> anyhow::Result<Vec<Alert>> {
        unimplemented!("To be implemented")
    }
}

impl AlertManager {
    pub async fn new() -> anyhow::Result<Self> {
        Ok(Self {
            alerts: std::sync::Arc::new(tokio::sync::Mutex::new(Vec::new())),
        })
    }
}