foxhunt/services/stress_tests/src/metrics.rs

//! Resilience and Recovery Metrics Collection
//!
//! Tracks recovery times, error rates, and system behavior under stress.

use hdrhistogram::Histogram;
use serde::{Deserialize, Serialize};
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::RwLock;

/// Recovery metrics for measuring system resilience
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RecoveryMetrics {
    /// Time taken to detect the failure
    pub detection_time: Duration,
    /// Time taken to recover from the failure
    pub recovery_time: Duration,
    /// Total downtime experienced
    pub total_downtime: Duration,
    /// Number of retry attempts before recovery
    pub retry_attempts: u32,
    /// Whether circuit breaker activated
    pub circuit_breaker_activated: bool,
    /// Whether graceful degradation occurred
    pub graceful_degradation: bool,
    /// Data consistency maintained during failure
    pub data_consistency_maintained: bool,
}

impl RecoveryMetrics {
    /// Create new recovery metrics
    pub fn new() -> Self {
        Self {
            detection_time: Duration::ZERO,
            recovery_time: Duration::ZERO,
            total_downtime: Duration::ZERO,
            retry_attempts: 0,
            circuit_breaker_activated: false,
            graceful_degradation: false,
            data_consistency_maintained: true,
        }
    }

    /// Calculate uptime percentage
    pub fn uptime_percentage(&self) -> f64 {
        let total_time = self.total_downtime + Duration::from_secs(1); // Avoid div by zero
        let uptime = total_time.saturating_sub(self.total_downtime);
        (uptime.as_secs_f64() / total_time.as_secs_f64()) * 100.0
    }

    /// Check if meets 99.9% uptime SLA
    pub fn meets_sla(&self, target_uptime: f64) -> bool {
        self.uptime_percentage() >= target_uptime
    }
}

impl Default for RecoveryMetrics {
    fn default() -> Self {
        Self::new()
    }
}

/// Comprehensive resilience metrics
pub struct ResilienceMetrics {
    /// Recovery time histogram (microseconds)
    recovery_times: Arc<RwLock<Histogram<u64>>>,
    /// Detection time histogram (microseconds)
    detection_times: Arc<RwLock<Histogram<u64>>>,
    /// Error count by category
    error_counts: Arc<RwLock<std::collections::HashMap<String, u64>>>,
    /// Circuit breaker activations
    circuit_breaker_count: Arc<RwLock<u64>>,
    /// Total test scenarios run
    total_scenarios: Arc<RwLock<u64>>,
    /// Successful recoveries
    successful_recoveries: Arc<RwLock<u64>>,
}

impl ResilienceMetrics {
    /// Create new resilience metrics collector
    pub fn new() -> Self {
        Self {
            recovery_times: Arc::new(RwLock::new(
                Histogram::<u64>::new(5).expect("Failed to create recovery histogram"),
            )),
            detection_times: Arc::new(RwLock::new(
                Histogram::<u64>::new(5).expect("Failed to create detection histogram"),
            )),
            error_counts: Arc::new(RwLock::new(std::collections::HashMap::new())),
            circuit_breaker_count: Arc::new(RwLock::new(0)),
            total_scenarios: Arc::new(RwLock::new(0)),
            successful_recoveries: Arc::new(RwLock::new(0)),
        }
    }

    /// Record recovery metrics from a test scenario
    pub async fn record_recovery(&self, metrics: &RecoveryMetrics) {
        // Record recovery time
        if self
            .recovery_times
            .write()
            .await
            .record(u64::try_from(metrics.recovery_time.as_micros()).unwrap_or(u64::MAX))
            .is_ok()
        {
            // Recorded successfully
        }

        // Record detection time
        if self
            .detection_times
            .write()
            .await
            .record(u64::try_from(metrics.detection_time.as_micros()).unwrap_or(u64::MAX))
            .is_ok()
        {
            // Recorded successfully
        }

        // Increment circuit breaker count if activated
        if metrics.circuit_breaker_activated {
            *self.circuit_breaker_count.write().await += 1;
        }

        // Increment successful recoveries if recovered
        if metrics.recovery_time > Duration::ZERO {
            *self.successful_recoveries.write().await += 1;
        }

        *self.total_scenarios.write().await += 1;
    }

    /// Record an error occurrence
    pub async fn record_error(&self, category: &str) {
        let mut counts = self.error_counts.write().await;
        *counts.entry(category.to_string()).or_insert(0) += 1;
    }

    /// Get mean recovery time
    pub async fn mean_recovery_time(&self) -> Duration {
        let hist = self.recovery_times.read().await;
        // hist.mean() returns f64, convert safely
        let mean_micros = hist.mean();
        let mean_u64 = mean_micros as u64;
        Duration::from_micros(mean_u64)
    }

    /// Get p99 recovery time
    pub async fn p99_recovery_time(&self) -> Duration {
        let hist = self.recovery_times.read().await;
        Duration::from_micros(hist.value_at_quantile(0.99))
    }

    /// Get success rate
    pub async fn success_rate(&self) -> f64 {
        let total = *self.total_scenarios.read().await as f64;
        if total == 0.0 {
            return 0.0;
        }
        let successful = *self.successful_recoveries.read().await as f64;
        (successful / total) * 100.0
    }

    /// Get circuit breaker activation rate
    pub async fn circuit_breaker_rate(&self) -> f64 {
        let total = *self.total_scenarios.read().await as f64;
        if total == 0.0 {
            return 0.0;
        }
        let activations = *self.circuit_breaker_count.read().await as f64;
        (activations / total) * 100.0
    }

    /// Generate summary report
    pub async fn generate_report(&self) -> String {
        let mean_recovery = self.mean_recovery_time().await;
        let p99_recovery = self.p99_recovery_time().await;
        let success_rate = self.success_rate().await;
        let cb_rate = self.circuit_breaker_rate().await;
        let total = *self.total_scenarios.read().await;

        format!(
            r#"
Resilience Metrics Summary
==========================
Total Scenarios: {}
Success Rate: {:.2}%
Circuit Breaker Activation Rate: {:.2}%

Recovery Times:
  Mean: {:?}
  P99:  {:?}

Error Counts:
"#,
            total, success_rate, cb_rate, mean_recovery, p99_recovery,
        )
    }
}

impl Default for ResilienceMetrics {
    fn default() -> Self {
        Self::new()
    }
}

/// Recovery timer for measuring recovery phases
pub struct RecoveryTimer {
    start: Instant,
    detection_time: Option<Duration>,
    recovery_time: Option<Duration>,
}

impl RecoveryTimer {
    /// Start a new recovery timer
    pub fn start() -> Self {
        Self {
            start: Instant::now(),
            detection_time: None,
            recovery_time: None,
        }
    }

    /// Mark failure detection
    pub fn mark_detection(&mut self) {
        self.detection_time = Some(self.start.elapsed());
    }

    /// Mark recovery completion
    pub fn mark_recovery(&mut self) {
        self.recovery_time = Some(self.start.elapsed());
    }

    /// Build recovery metrics
    pub fn build_metrics(self) -> RecoveryMetrics {
        RecoveryMetrics {
            detection_time: self.detection_time.unwrap_or(Duration::ZERO),
            recovery_time: self.recovery_time.unwrap_or(Duration::ZERO),
            total_downtime: self.recovery_time.unwrap_or(Duration::ZERO),
            retry_attempts: 0,
            circuit_breaker_activated: false,
            graceful_degradation: false,
            data_consistency_maintained: true,
        }
    }
}