foxhunt/crates/ml-ensemble/src/metrics.rs

//! Prometheus metrics for ensemble hot-swapping
//!
//! This module provides comprehensive metrics for monitoring checkpoint
//! hot-swapping operations, including swap success rates, latencies,
//! validation results, and canary monitoring.

use once_cell::sync::Lazy;
use prometheus::{register_counter_vec, register_histogram_vec, CounterVec, HistogramVec};

// Metric registration is infallible in practice; once_cell::Lazy closures cannot use `?`
#[allow(clippy::expect_used)]
/// Counter for checkpoint swaps by status
pub static CHECKPOINT_SWAPS_TOTAL: Lazy<CounterVec> = Lazy::new(|| {
    register_counter_vec!(
        "checkpoint_swaps_total",
        "Total checkpoint hot-swaps by model and status",
        &["model_id", "status"] // status: success, rollback, failed
    )
    .expect("Failed to register checkpoint_swaps_total")
});

#[allow(clippy::expect_used)]
/// Histogram for checkpoint swap latency
pub static CHECKPOINT_SWAP_LATENCY_MICROSECONDS: Lazy<HistogramVec> = Lazy::new(|| {
    register_histogram_vec!(
        "checkpoint_swap_latency_microseconds",
        "Checkpoint swap latency in microseconds (atomic swap operation)",
        &["model_id"],
        vec![0.1, 0.5, 1.0, 5.0, 10.0, 50.0, 100.0] // Sub-microsecond to 100μs
    )
    .expect("Failed to register checkpoint_swap_latency_microseconds")
});

#[allow(clippy::expect_used)]
/// Counter for checkpoint validations
pub static CHECKPOINT_VALIDATION_TOTAL: Lazy<CounterVec> = Lazy::new(|| {
    register_counter_vec!(
        "checkpoint_validation_total",
        "Total checkpoint validations by model and result",
        &["model_id", "result"] // result: passed, failed
    )
    .expect("Failed to register checkpoint_validation_total")
});

#[allow(clippy::expect_used)]
/// Histogram for checkpoint validation latency
pub static CHECKPOINT_VALIDATION_LATENCY_MILLISECONDS: Lazy<HistogramVec> = Lazy::new(|| {
    register_histogram_vec!(
        "checkpoint_validation_latency_milliseconds",
        "Checkpoint validation latency in milliseconds (1000 predictions)",
        &["model_id"],
        vec![10.0, 50.0, 100.0, 500.0, 1000.0, 5000.0]
    )
    .expect("Failed to register checkpoint_validation_latency_milliseconds")
});

#[allow(clippy::expect_used)]
/// Histogram for validated checkpoint P99 inference latency
pub static CHECKPOINT_P99_LATENCY_MICROSECONDS: Lazy<HistogramVec> = Lazy::new(|| {
    register_histogram_vec!(
        "checkpoint_p99_latency_microseconds",
        "P99 inference latency from checkpoint validation",
        &["model_id"],
        vec![1.0, 5.0, 10.0, 25.0, 50.0, 100.0]
    )
    .expect("Failed to register checkpoint_p99_latency_microseconds")
});

#[allow(clippy::expect_used)]
/// Counter for canary monitoring results
pub static CANARY_MONITORING_TOTAL: Lazy<CounterVec> = Lazy::new(|| {
    register_counter_vec!(
        "canary_monitoring_total",
        "Total canary monitoring runs by model and result",
        &["model_id", "result"] // result: success, failed
    )
    .expect("Failed to register canary_monitoring_total")
});

#[allow(clippy::expect_used)]
/// Histogram for canary monitoring duration
pub static CANARY_MONITORING_DURATION_SECONDS: Lazy<HistogramVec> = Lazy::new(|| {
    register_histogram_vec!(
        "canary_monitoring_duration_seconds",
        "Canary monitoring duration in seconds",
        &["model_id"],
        vec![60.0, 120.0, 300.0, 600.0, 1800.0] // 1 min to 30 min
    )
    .expect("Failed to register canary_monitoring_duration_seconds")
});

#[allow(clippy::expect_used)]
/// Counter for rollbacks by reason
pub static CHECKPOINT_ROLLBACKS_TOTAL: Lazy<CounterVec> = Lazy::new(|| {
    register_counter_vec!(
        "checkpoint_rollbacks_total",
        "Total checkpoint rollbacks by model and reason",
        &["model_id", "reason"] // reason: latency, error_rate, accuracy_drop
    )
    .expect("Failed to register checkpoint_rollbacks_total")
});

/// Ensemble metrics collector
#[derive(Debug)]
pub struct EnsembleMetrics;

impl EnsembleMetrics {
    /// Record successful checkpoint swap
    pub fn record_swap_success(model_id: &str, latency_us: u64) {
        CHECKPOINT_SWAPS_TOTAL
            .with_label_values(&[model_id, "success"])
            .inc();

        CHECKPOINT_SWAP_LATENCY_MICROSECONDS
            .with_label_values(&[model_id])
            .observe(latency_us as f64);
    }

    /// Record checkpoint swap rollback
    pub fn record_swap_rollback(model_id: &str) {
        CHECKPOINT_SWAPS_TOTAL
            .with_label_values(&[model_id, "rollback"])
            .inc();
    }

    /// Record checkpoint swap failure
    pub fn record_swap_failed(model_id: &str) {
        CHECKPOINT_SWAPS_TOTAL
            .with_label_values(&[model_id, "failed"])
            .inc();
    }

    /// Record checkpoint validation result
    pub fn record_validation(
        model_id: &str,
        passed: bool,
        validation_latency_ms: u64,
        p99_latency_us: u64,
    ) {
        let result = if passed { "passed" } else { "failed" };

        CHECKPOINT_VALIDATION_TOTAL
            .with_label_values(&[model_id, result])
            .inc();

        CHECKPOINT_VALIDATION_LATENCY_MILLISECONDS
            .with_label_values(&[model_id])
            .observe(validation_latency_ms as f64);

        CHECKPOINT_P99_LATENCY_MICROSECONDS
            .with_label_values(&[model_id])
            .observe(p99_latency_us as f64);
    }

    /// Record canary monitoring result
    pub fn record_canary(model_id: &str, success: bool, duration_secs: u64) {
        let result = if success { "success" } else { "failed" };

        CANARY_MONITORING_TOTAL
            .with_label_values(&[model_id, result])
            .inc();

        CANARY_MONITORING_DURATION_SECONDS
            .with_label_values(&[model_id])
            .observe(duration_secs as f64);
    }

    /// Record rollback with reason
    pub fn record_rollback(model_id: &str, reason: &str) {
        CHECKPOINT_ROLLBACKS_TOTAL
            .with_label_values(&[model_id, reason])
            .inc();
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use tracing::info;

    #[test]
    fn test_metrics_recording() {
        // Record successful swap
        EnsembleMetrics::record_swap_success("DQN", 500);

        // Record validation
        EnsembleMetrics::record_validation("DQN", true, 150, 35);

        // Record canary success
        EnsembleMetrics::record_canary("DQN", true, 300);

        // Record rollback
        EnsembleMetrics::record_rollback("DQN", "latency");

        // Verify metrics were recorded (metrics are global, just test they don't panic)
        info!("All metrics recorded successfully");
    }
}