foxhunt/crates/ml-observability/src/alerts.rs

//! Alert management system for ML production monitoring

use anyhow::Result;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use tokio::sync::RwLock;

/// Alert severity levels
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum AlertSeverity {
    Info,
    Warning,
    Critical,
    Emergency,
}

/// Alert channels for notification delivery
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum AlertChannel {
    Slack { webhook_url: String },
    Email { recipients: Vec<String> },
    PagerDuty { service_key: String },
    Webhook { url: String },
    Console,
}

/// Alert rule configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AlertRule {
    pub name: String,
    pub metric_name: String,
    pub threshold: f64,
    pub comparison: AlertComparison,
    pub severity: AlertSeverity,
    pub cooldown_minutes: u64,
    pub channels: Vec<AlertChannel>,
    pub labels: HashMap<String, String>,
}

#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub enum AlertComparison {
    GreaterThan,
    LessThan,
    Equal,
    NotEqual,
}

/// Active alert
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Alert {
    pub id: String,
    pub rule_name: String,
    pub metric_name: String,
    pub current_value: f64,
    pub threshold: f64,
    pub severity: AlertSeverity,
    pub message: String,
    pub labels: HashMap<String, String>,
    pub triggered_at: SystemTime,
    pub acknowledged: bool,
    pub resolved: bool,
}

/// Alert management system
#[derive(Debug)]
pub struct AlertManager {
    rules: Arc<RwLock<Vec<AlertRule>>>,
    active_alerts: Arc<RwLock<HashMap<String, Alert>>>,
    cooldown_tracker: Arc<RwLock<HashMap<String, SystemTime>>>,
}

impl AlertManager {
    pub fn new() -> Self {
        Self {
            rules: Arc::new(RwLock::new(Vec::new())),
            active_alerts: Arc::new(RwLock::new(HashMap::new())),
            cooldown_tracker: Arc::new(RwLock::new(HashMap::new())),
        }
    }

    /// Add alert rule
    pub async fn add_rule(&self, rule: AlertRule) {
        let mut rules = self.rules.write().await;
        rules.push(rule);
    }

    /// Evaluate metric against all rules
    pub async fn evaluate_metric(
        &self,
        metric_name: &str,
        value: f64,
        labels: HashMap<String, String>,
    ) -> Result<()> {
        let rules = self.rules.read().await;

        for rule in rules.iter() {
            if rule.metric_name == metric_name {
                self.evaluate_rule(rule, value, labels.clone()).await?;
            }
        }

        Ok(())
    }

    async fn evaluate_rule(
        &self,
        rule: &AlertRule,
        value: f64,
        labels: HashMap<String, String>,
    ) -> Result<()> {
        let should_trigger = match rule.comparison {
            AlertComparison::GreaterThan => value > rule.threshold,
            AlertComparison::LessThan => value < rule.threshold,
            AlertComparison::Equal => (value - rule.threshold).abs() < f64::EPSILON,
            AlertComparison::NotEqual => (value - rule.threshold).abs() > f64::EPSILON,
        };

        if should_trigger {
            self.trigger_alert(rule, value, labels).await?;
        }

        Ok(())
    }

    async fn trigger_alert(
        &self,
        rule: &AlertRule,
        value: f64,
        labels: HashMap<String, String>,
    ) -> Result<()> {
        let alert_id = format!(
            "{}_{}",
            rule.name,
            SystemTime::now()
                .duration_since(SystemTime::UNIX_EPOCH)?
                .as_secs()
        );

        // Check cooldown
        {
            let cooldown = self.cooldown_tracker.read().await;
            if let Some(last_trigger) = cooldown.get(&rule.name) {
                let elapsed = SystemTime::now().duration_since(*last_trigger)?;
                if elapsed < Duration::from_secs(rule.cooldown_minutes * 60) {
                    return Ok(()); // Still in cooldown
                }
            }
        }

        let alert = Alert {
            id: alert_id.clone(),
            rule_name: rule.name.clone(),
            metric_name: rule.metric_name.clone(),
            current_value: value,
            threshold: rule.threshold,
            severity: rule.severity,
            message: format!(
                "Alert: {} - {} {} {} (current: {})",
                rule.name,
                rule.metric_name,
                match rule.comparison {
                    AlertComparison::GreaterThan => ">",
                    AlertComparison::LessThan => "<",
                    AlertComparison::Equal => "==",
                    AlertComparison::NotEqual => "!=",
                },
                rule.threshold,
                value
            ),
            labels,
            triggered_at: SystemTime::now(),
            acknowledged: false,
            resolved: false,
        };

        // Store alert
        {
            let mut alerts = self.active_alerts.write().await;
            alerts.insert(alert_id, alert.clone());
        }

        // Update cooldown
        {
            let mut cooldown = self.cooldown_tracker.write().await;
            cooldown.insert(rule.name.clone(), SystemTime::now());
        }

        // Send notifications
        for channel in &rule.channels {
            self.send_notification(channel, &alert).await?;
        }

        tracing::warn!("Alert triggered: {}", alert.message);
        Ok(())
    }

    async fn send_notification(&self, channel: &AlertChannel, alert: &Alert) -> Result<()> {
        match channel {
            AlertChannel::Console => {
                println!("ALERT: {} - {}", alert.severity as u8, alert.message);
            },
            AlertChannel::Slack { webhook_url: _ } => {
                // Implement Slack webhook notification
                tracing::info!("Would send Slack alert: {}", alert.message);
            },
            AlertChannel::Email { recipients: _ } => {
                // Implement email notification
                tracing::info!("Would send email alert: {}", alert.message);
            },
            AlertChannel::PagerDuty { service_key: _ } => {
                // Implement PagerDuty notification
                tracing::info!("Would send PagerDuty alert: {}", alert.message);
            },
            AlertChannel::Webhook { url: _ } => {
                // Implement webhook notification
                tracing::info!("Would send webhook alert: {}", alert.message);
            },
        }
        Ok(())
    }

    /// Get active alerts
    pub async fn get_active_alerts(&self) -> HashMap<String, Alert> {
        self.active_alerts.read().await.clone()
    }

    /// Acknowledge alert
    pub async fn acknowledge_alert(&self, alert_id: &str) -> Result<()> {
        let mut alerts = self.active_alerts.write().await;
        if let Some(alert) = alerts.get_mut(alert_id) {
            alert.acknowledged = true;
            tracing::info!("Alert acknowledged: {}", alert_id);
        }
        Ok(())
    }

    /// Resolve alert
    pub async fn resolve_alert(&self, alert_id: &str) -> Result<()> {
        let mut alerts = self.active_alerts.write().await;
        if let Some(alert) = alerts.get_mut(alert_id) {
            alert.resolved = true;
            tracing::info!("Alert resolved: {}", alert_id);
        }
        Ok(())
    }
}

/// Create default HFT alert rules
pub fn create_hft_alert_rules() -> Vec<AlertRule> {
    vec![
        AlertRule {
            name: "high_inference_latency".to_owned(),
            metric_name: "ml_inference_latency_microseconds".to_owned(),
            threshold: 100.0, // 100us
            comparison: AlertComparison::GreaterThan,
            severity: AlertSeverity::Warning,
            cooldown_minutes: 5,
            channels: vec![AlertChannel::Console],
            labels: HashMap::new(),
        },
        AlertRule {
            name: "critical_inference_latency".to_owned(),
            metric_name: "ml_inference_latency_microseconds".to_owned(),
            threshold: 500.0, // 500us
            comparison: AlertComparison::GreaterThan,
            severity: AlertSeverity::Critical,
            cooldown_minutes: 1,
            channels: vec![AlertChannel::Console],
            labels: HashMap::new(),
        },
        AlertRule {
            name: "high_error_rate".to_owned(),
            metric_name: "ml_error_rate".to_owned(),
            threshold: 0.05, // 5%
            comparison: AlertComparison::GreaterThan,
            severity: AlertSeverity::Warning,
            cooldown_minutes: 10,
            channels: vec![AlertChannel::Console],
            labels: HashMap::new(),
        },
        AlertRule {
            name: "low_model_confidence".to_owned(),
            metric_name: "ml_model_confidence".to_owned(),
            threshold: 0.6, // 60%
            comparison: AlertComparison::LessThan,
            severity: AlertSeverity::Warning,
            cooldown_minutes: 15,
            channels: vec![AlertChannel::Console],
            labels: HashMap::new(),
        },
        AlertRule {
            name: "model_drift_detected".to_owned(),
            metric_name: "ml_drift_detection_score".to_owned(),
            threshold: 0.3,
            comparison: AlertComparison::GreaterThan,
            severity: AlertSeverity::Warning,
            cooldown_minutes: 30,
            channels: vec![AlertChannel::Console],
            labels: HashMap::new(),
        },
    ]
}

impl Default for AlertManager {
    fn default() -> Self {
        Self::new()
    }
}