foxhunt/testing/integration/chaos/nightly_chaos_runner.rs

//! Nightly Chaos Job Automation
//!
//! Automated scheduling and execution of chaos engineering tests
//! for continuous validation of system resilience.

use anyhow::{Context, Result};
use chrono::{Datelike, DateTime, NaiveTime, Utc};
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use tokio::fs;
use tokio::sync::{broadcast, RwLock};
use tokio::time::{interval, sleep_until, timeout, Instant};
use tracing::{error, info, warn};
use uuid::Uuid;

use super::chaos_framework::ChaosResult;
use super::ml_training_chaos::{MLChaosConfig, MLChaosResult, MLTrainingChaosTests};

/// Nightly chaos job configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NightlyChaosConfig {
    pub enabled: bool,
    pub schedule_time: NaiveTime, // Time to run chaos tests (e.g., 2:00 AM)
    pub timezone: String,         // Timezone for scheduling (e.g., "UTC", "America/New_York")
    pub max_duration_hours: u8,   // Maximum time chaos tests can run
    pub notification_webhook: Option<String>, // Slack/Teams webhook for alerts
    pub report_storage_path: PathBuf,
    pub ml_chaos_config: MLChaosConfig,
    pub exclude_weekends: bool,
    pub retry_on_failure: bool,
    pub max_retries: u8,
}

impl Default for NightlyChaosConfig {
    fn default() -> Self {
        Self {
            enabled: true,
            schedule_time: NaiveTime::from_hms_opt(2, 0, 0).unwrap(), // 2:00 AM
            timezone: "UTC".to_string(),
            max_duration_hours: 4,
            notification_webhook: None,
            report_storage_path: PathBuf::from("/tmp/chaos_reports"),
            ml_chaos_config: MLChaosConfig {
                ml_service_endpoint: "http://localhost:8080".to_string(),
                checkpoint_base_path: PathBuf::from("/tmp/ml_checkpoints"),
                model_types: vec![
                    super::ml_training_chaos::ModelType::TLOB,
                    super::ml_training_chaos::ModelType::DQN,
                    super::ml_training_chaos::ModelType::MAMBA2,
                ],
                training_timeout_secs: 300,
                max_recovery_time_ms: 100, // HFT requirement
                gpu_memory_threshold_mb: 8192,
            },
            exclude_weekends: true,
            retry_on_failure: true,
            max_retries: 2,
        }
    }
}

/// Chaos job execution status
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ChaosJobStatus {
    Scheduled,
    Running,
    Completed,
    Failed,
    Cancelled,
    Retrying { attempt: u8 },
}

/// Chaos job execution record
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChaosJobExecution {
    pub id: Uuid,
    pub scheduled_at: DateTime<Utc>,
    pub started_at: Option<DateTime<Utc>>,
    pub completed_at: Option<DateTime<Utc>>,
    pub status: ChaosJobStatus,
    pub chaos_results: Vec<ChaosResult>,
    pub ml_chaos_results: Vec<MLChaosResult>,
    pub total_experiments: usize,
    pub successful_experiments: usize,
    pub failed_experiments: usize,
    pub report_path: Option<PathBuf>,
    pub error_messages: Vec<String>,
    pub performance_summary: Option<PerformanceSummary>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PerformanceSummary {
    pub total_recovery_time_ms: u64,
    pub average_recovery_time_ms: f64,
    pub max_recovery_time_ms: u64,
    pub sla_violations: usize, // Recovery times > max_recovery_time_ms
    pub performance_regressions: usize,
    pub checkpoint_failures: usize,
}

/// Nightly chaos job scheduler and runner
pub struct NightlyChaosRunner {
    config: Arc<RwLock<NightlyChaosConfig>>,
    job_history: Arc<RwLock<Vec<ChaosJobExecution>>>,
    _event_sender: broadcast::Sender<ChaosJobEvent>,
    is_running: Arc<RwLock<bool>>,
}

#[derive(Debug, Clone)]
pub enum ChaosJobEvent {
    JobScheduled {
        id: Uuid,
        scheduled_time: DateTime<Utc>,
    },
    JobStarted {
        id: Uuid,
    },
    JobCompleted {
        id: Uuid,
        summary: PerformanceSummary,
    },
    JobFailed {
        id: Uuid,
        error: String,
    },
    JobRetrying {
        id: Uuid,
        attempt: u8,
    },
    AlertTriggered {
        message: String,
        severity: AlertSeverity,
    },
}

#[derive(Debug, Clone)]
pub enum AlertSeverity {
    Info,
    Warning,
    Critical,
}

impl NightlyChaosRunner {
    pub fn new(config: NightlyChaosConfig) -> Self {
        let (_event_sender, _) = broadcast::channel(1000);

        Self {
            config: Arc::new(RwLock::new(config)),
            job_history: Arc::new(RwLock::new(Vec::new())),
            _event_sender,
            is_running: Arc::new(RwLock::new(false)),
        }
    }

    /// Start the nightly chaos job scheduler
    pub async fn start(&self) -> Result<()> {
        {
            let mut running = self.is_running.write().await;
            if *running {
                return Err(anyhow::anyhow!("Chaos runner is already running"));
            }
            *running = true;
        }

        info!("Starting nightly chaos job scheduler");

        let config = self.config.read().await.clone();
        if !config.enabled {
            warn!("Nightly chaos jobs are disabled in configuration");
            return Ok(());
        }

        // Create report storage directory
        fs::create_dir_all(&config.report_storage_path)
            .await
            .context("Failed to create report storage directory")?;

        // Start scheduling loop
        let scheduler_handle = {
            let runner = self.clone();
            tokio::spawn(async move {
                runner.scheduling_loop().await;
            })
        };

        // Wait for the scheduler (it runs indefinitely)
        let _ = scheduler_handle.await;

        Ok(())
    }

    /// Stop the chaos job scheduler
    pub async fn stop(&self) {
        let mut running = self.is_running.write().await;
        *running = false;
        info!("Stopped nightly chaos job scheduler");
    }

    /// Main scheduling loop
    async fn scheduling_loop(&self) {
        let mut check_interval = interval(Duration::from_secs(60)); // Check every minute

        loop {
            check_interval.tick().await;

            // Check if we should stop
            {
                let running = self.is_running.read().await;
                if !*running {
                    break;
                }
            }

            let config = self.config.read().await.clone();
            if !config.enabled {
                continue;
            }

            // Check if it's time to run chaos tests
            if self.should_run_chaos_tests(&config).await {
                match self.schedule_chaos_job().await {
                    Ok(job_id) => {
                        info!("Scheduled chaos job: {}", job_id);

                        // Execute the job
                        let runner = self.clone();
                        tokio::spawn(async move {
                            if let Err(e) = runner.execute_chaos_job(job_id).await {
                                error!("Chaos job execution failed: {}", e);
                            }
                        });
                    }
                    Err(e) => {
                        error!("Failed to schedule chaos job: {}", e);
                    }
                }

                // Wait 24 hours before next check to avoid duplicate runs
                sleep_until(Instant::now() + Duration::from_secs(24 * 60 * 60)).await;
            }
        }
    }

    /// Check if chaos tests should run now
    async fn should_run_chaos_tests(&self, config: &NightlyChaosConfig) -> bool {
        let now = Utc::now();

        // Skip weekends if configured
        if config.exclude_weekends {
            let weekday = now.weekday();
            if weekday == chrono::Weekday::Sat || weekday == chrono::Weekday::Sun {
                return false;
            }
        }

        // Check if it's the scheduled time (within 1 minute window)
        let current_time = now.time();
        let schedule_time = config.schedule_time;

        let diff = if current_time >= schedule_time {
            current_time - schedule_time
        } else {
            // Handle day boundary
            chrono::NaiveTime::from_hms_opt(23, 59, 59).unwrap() - schedule_time
                + chrono::Duration::seconds(60)
                + (current_time - chrono::NaiveTime::from_hms_opt(0, 0, 0).unwrap())
        };

        // Run if within 1 minute of scheduled time
        diff <= Duration::minutes(1)
    }

    /// Schedule a new chaos job
    async fn schedule_chaos_job(&self) -> Result<Uuid> {
        let job_id = Uuid::new_v4();
        let now = Utc::now();

        let job = ChaosJobExecution {
            id: job_id,
            scheduled_at: now,
            started_at: None,
            completed_at: None,
            status: ChaosJobStatus::Scheduled,
            chaos_results: Vec::new(),
            ml_chaos_results: Vec::new(),
            total_experiments: 0,
            successful_experiments: 0,
            failed_experiments: 0,
            report_path: None,
            error_messages: Vec::new(),
            performance_summary: None,
        };

        // Add to job history
        {
            let mut history = self.job_history.write().await;
            history.push(job.clone());
        }

        // Send scheduling event
        let _ = self._event_sender.send(ChaosJobEvent::JobScheduled {
            id: job_id,
            scheduled_time: now,
        });

        Ok(job_id)
    }

    /// Execute a chaos job
    async fn execute_chaos_job(&self, job_id: Uuid) -> Result<()> {
        info!("Executing chaos job: {}", job_id);

        // Update job status
        self.update_job_status(job_id, ChaosJobStatus::Running)
            .await?;
        self.update_job_start_time(job_id, Some(Utc::now())).await?;

        // Send start event
        let _ = self
            ._event_sender
            .send(ChaosJobEvent::JobStarted { id: job_id });

        let config = self.config.read().await.clone();
        let mut attempt = 1;

        loop {
            match self.run_chaos_experiments(job_id, &config).await {
                Ok(summary) => {
                    // Job succeeded
                    self.update_job_status(job_id, ChaosJobStatus::Completed)
                        .await?;
                    self.update_job_completion_time(job_id, Some(Utc::now()))
                        .await?;
                    self.update_job_performance_summary(job_id, Some(summary.clone()))
                        .await?;

                    // Generate and save report
                    if let Err(e) = self.generate_and_save_report(job_id).await {
                        error!("Failed to generate chaos job report: {}", e);
                    }

                    // Send completion event
                    let _ = self._event_sender.send(ChaosJobEvent::JobCompleted {
                        id: job_id,
                        summary: summary.clone(),
                    });

                    // Send alerts if needed
                    self.check_and_send_alerts(job_id, &summary).await;

                    info!("Chaos job completed successfully: {}", job_id);
                    break;
                }
                Err(e) => {
                    error!("Chaos job failed (attempt {}): {}", attempt, e);

                    if config.retry_on_failure && attempt <= config.max_retries {
                        // Retry the job
                        self.update_job_status(job_id, ChaosJobStatus::Retrying { attempt })
                            .await?;
                        self.add_job_error(job_id, format!("Attempt {} failed: {}", attempt, e))
                            .await?;

                        let _ = self._event_sender.send(ChaosJobEvent::JobRetrying {
                            id: job_id,
                            attempt,
                        });

                        attempt += 1;
                        sleep_until(Instant::now() + Duration::from_secs(300)).await; // Wait 5 minutes before retry
                        continue;
                    } else {
                        // Job failed permanently
                        self.update_job_status(job_id, ChaosJobStatus::Failed)
                            .await?;
                        self.update_job_completion_time(job_id, Some(Utc::now()))
                            .await?;
                        self.add_job_error(job_id, e.to_string()).await?;

                        let _ = self._event_sender.send(ChaosJobEvent::JobFailed {
                            id: job_id,
                            error: e.to_string(),
                        });

                        error!("Chaos job failed permanently: {}", job_id);
                        break;
                    }
                }
            }
        }

        Ok(())
    }

    /// Run all chaos experiments
    async fn run_chaos_experiments(
        &self,
        job_id: Uuid,
        config: &NightlyChaosConfig,
    ) -> Result<PerformanceSummary> {
        info!("Running chaos experiments for job: {}", job_id);

        // Initialize ML chaos tests
        let ml_chaos_tests = MLTrainingChaosTests::new(config.ml_chaos_config.clone());

        // Run ML-specific chaos experiments
        let ml_results = timeout(
            Duration::from_secs(config.max_duration_hours as u64 * 3600),
            ml_chaos_tests.run_ml_chaos_suite(),
        )
        .await
        .context("ML chaos tests timed out")?
        .context("ML chaos tests failed")?;

        // Update job with ML results
        self.update_job_ml_results(job_id, ml_results.clone())
            .await?;

        // Calculate performance summary
        let summary = self.calculate_performance_summary(&[], &ml_results);

        Ok(summary)
    }

    /// Calculate performance summary from results
    fn calculate_performance_summary(
        &self,
        chaos_results: &[ChaosResult],
        ml_results: &[MLChaosResult],
    ) -> PerformanceSummary {
        let mut total_recovery_time_ms = 0u64;
        let mut recovery_times = Vec::new();
        let mut sla_violations = 0;
        let mut performance_regressions = 0;
        let mut checkpoint_failures = 0;

        // Process general chaos results
        for result in chaos_results {
            if let Some(recovery_time) = result.recovery_time_ms {
                total_recovery_time_ms += recovery_time;
                recovery_times.push(recovery_time);

                // Check for SLA violations (assuming 100ms max recovery time for HFT)
                if recovery_time > 100 {
                    sla_violations += 1;
                }
            }

            if result.performance_regression.is_some() {
                performance_regressions += 1;
            }

            if !result.checkpoint_integrity {
                checkpoint_failures += 1;
            }
        }

        // Process ML-specific results
        for ml_result in ml_results {
            if !ml_result.training_loss_continuity {
                checkpoint_failures += 1;
            }

            if ml_result.performance_regression.is_some() {
                performance_regressions += 1;
            }
        }

        let average_recovery_time_ms = if !recovery_times.is_empty() {
            total_recovery_time_ms as f64 / recovery_times.len() as f64
        } else {
            0.0
        };

        let max_recovery_time_ms = recovery_times.into_iter().max().unwrap_or(0);

        PerformanceSummary {
            total_recovery_time_ms,
            average_recovery_time_ms,
            max_recovery_time_ms,
            sla_violations,
            performance_regressions,
            checkpoint_failures,
        }
    }

    /// Generate and save chaos job report
    async fn generate_and_save_report(&self, job_id: Uuid) -> Result<()> {
        let job = {
            let history = self.job_history.read().await;
            history
                .iter()
                .find(|j| j.id == job_id)
                .cloned()
                .ok_or_else(|| anyhow::anyhow!("Job not found: {}", job_id))?
        };

        let config = self.config.read().await.clone();

        // Generate ML chaos report
        let ml_chaos_tests = MLTrainingChaosTests::new(config.ml_chaos_config.clone());
        let ml_report = ml_chaos_tests
            .generate_chaos_report(&job.ml_chaos_results)
            .await?;

        // Generate comprehensive report
        let mut full_report = String::new();
        full_report.push_str("# Nightly Chaos Engineering Report\n\n");
        full_report.push_str(&format!("**Job ID:** {}\n", job.id));
        full_report.push_str(&format!("**Scheduled:** {}\n", job.scheduled_at));
        full_report.push_str(&format!(
            "**Started:** {}\n",
            job.started_at.map_or("N/A".to_string(), |t| t.to_string())
        ));
        full_report.push_str(&format!(
            "**Completed:** {}\n",
            job.completed_at
                .map_or("N/A".to_string(), |t| t.to_string())
        ));
        full_report.push_str(&format!("**Status:** {:?}\n\n", job.status));

        // Performance summary
        if let Some(ref summary) = job.performance_summary {
            full_report.push_str("## Performance Summary\n");
            full_report.push_str(&format!(
                "- **Average Recovery Time:** {:.1}ms\n",
                summary.average_recovery_time_ms
            ));
            full_report.push_str(&format!(
                "- **Max Recovery Time:** {}ms\n",
                summary.max_recovery_time_ms
            ));
            full_report.push_str(&format!(
                "- **SLA Violations:** {}\n",
                summary.sla_violations
            ));
            full_report.push_str(&format!(
                "- **Performance Regressions:** {}\n",
                summary.performance_regressions
            ));
            full_report.push_str(&format!(
                "- **Checkpoint Failures:** {}\n\n",
                summary.checkpoint_failures
            ));
        }

        // Add ML-specific report
        full_report.push_str(&ml_report);

        // Add error messages if any
        if !job.error_messages.is_empty() {
            full_report.push_str("\n## Error Messages\n");
            for error in &job.error_messages {
                full_report.push_str(&format!("- {}\n", error));
            }
        }

        // Save report to file
        let report_filename = format!(
            "chaos_report_{}_{}.md",
            job_id,
            job.scheduled_at.format("%Y%m%d_%H%M%S")
        );
        let report_path = config.report_storage_path.join(report_filename);

        fs::write(&report_path, full_report)
            .await
            .context("Failed to save chaos report")?;

        // Update job with report path
        self.update_job_report_path(job_id, Some(report_path))
            .await?;

        Ok(())
    }

    /// Check for alerts and send notifications
    async fn check_and_send_alerts(&self, job_id: Uuid, summary: &PerformanceSummary) {
        let config = self.config.read().await.clone();

        // Check for critical alerts
        if summary.sla_violations > 0 {
            let message = format!(
                "🚨 CRITICAL: {} SLA violations detected in chaos job {}. Max recovery time: {}ms",
                summary.sla_violations, job_id, summary.max_recovery_time_ms
            );

            let _ = self._event_sender.send(ChaosJobEvent::AlertTriggered {
                message: message.clone(),
                severity: AlertSeverity::Critical,
            });

            if let Some(ref webhook) = config.notification_webhook {
                self.send_webhook_notification(webhook, &message, AlertSeverity::Critical)
                    .await;
            }
        }

        // Check for warning alerts
        if summary.checkpoint_failures > 0 {
            let message = format!(
                "⚠️ WARNING: {} checkpoint failures detected in chaos job {}",
                summary.checkpoint_failures, job_id
            );

            let _ = self._event_sender.send(ChaosJobEvent::AlertTriggered {
                message: message.clone(),
                severity: AlertSeverity::Warning,
            });

            if let Some(ref webhook) = config.notification_webhook {
                self.send_webhook_notification(webhook, &message, AlertSeverity::Warning)
                    .await;
            }
        }

        // Send success notification
        if summary.sla_violations == 0 && summary.checkpoint_failures == 0 {
            let message = format!(
                "✅ Chaos job {} completed successfully. Avg recovery time: {:.1}ms",
                job_id, summary.average_recovery_time_ms
            );

            if let Some(ref webhook) = config.notification_webhook {
                self.send_webhook_notification(webhook, &message, AlertSeverity::Info)
                    .await;
            }
        }
    }

    /// Send webhook notification
    async fn send_webhook_notification(
        &self,
        webhook_url: &str,
        message: &str,
        severity: AlertSeverity,
    ) {
        // TODO: Implement actual webhook sending (Slack, Teams, etc.)
        info!(
            "Sending {} alert: {}",
            match severity {
                AlertSeverity::Info => "INFO",
                AlertSeverity::Warning => "WARNING",
                AlertSeverity::Critical => "CRITICAL",
            },
            message
        );
    }

    /// Subscribe to chaos job events
    pub fn subscribe_events(&self) -> broadcast::Receiver<ChaosJobEvent> {
        self._event_sender.subscribe()
    }

    /// Get job history
    pub async fn get_job_history(&self) -> Vec<ChaosJobExecution> {
        self.job_history.read().await.clone()
    }

    /// Update configuration
    pub async fn update_config(&self, new_config: NightlyChaosConfig) {
        let mut config = self.config.write().await;
        *config = new_config;
        info!("Updated nightly chaos configuration");
    }

    // Helper methods for updating job state
    async fn update_job_status(&self, job_id: Uuid, status: ChaosJobStatus) -> Result<()> {
        let mut history = self.job_history.write().await;
        if let Some(job) = history.iter_mut().find(|j| j.id == job_id) {
            job.status = status;
        }
        Ok(())
    }

    async fn update_job_start_time(
        &self,
        job_id: Uuid,
        start_time: Option<DateTime<Utc>>,
    ) -> Result<()> {
        let mut history = self.job_history.write().await;
        if let Some(job) = history.iter_mut().find(|j| j.id == job_id) {
            job.started_at = start_time;
        }
        Ok(())
    }

    async fn update_job_completion_time(
        &self,
        job_id: Uuid,
        completion_time: Option<DateTime<Utc>>,
    ) -> Result<()> {
        let mut history = self.job_history.write().await;
        if let Some(job) = history.iter_mut().find(|j| j.id == job_id) {
            job.completed_at = completion_time;
        }
        Ok(())
    }

    async fn update_job_ml_results(
        &self,
        job_id: Uuid,
        ml_results: Vec<MLChaosResult>,
    ) -> Result<()> {
        let mut history = self.job_history.write().await;
        if let Some(job) = history.iter_mut().find(|j| j.id == job_id) {
            job.ml_chaos_results = ml_results;
            job.total_experiments = job.chaos_results.len() + job.ml_chaos_results.len();
            job.successful_experiments = job
                .ml_chaos_results
                .iter()
                .filter(|r| r.training_loss_continuity)
                .count();
            job.failed_experiments = job.total_experiments - job.successful_experiments;
        }
        Ok(())
    }

    async fn update_job_performance_summary(
        &self,
        job_id: Uuid,
        summary: Option<PerformanceSummary>,
    ) -> Result<()> {
        let mut history = self.job_history.write().await;
        if let Some(job) = history.iter_mut().find(|j| j.id == job_id) {
            job.performance_summary = summary;
        }
        Ok(())
    }

    async fn update_job_report_path(
        &self,
        job_id: Uuid,
        report_path: Option<PathBuf>,
    ) -> Result<()> {
        let mut history = self.job_history.write().await;
        if let Some(job) = history.iter_mut().find(|j| j.id == job_id) {
            job.report_path = report_path;
        }
        Ok(())
    }

    async fn add_job_error(&self, job_id: Uuid, error: String) -> Result<()> {
        let mut history = self.job_history.write().await;
        if let Some(job) = history.iter_mut().find(|j| j.id == job_id) {
            job.error_messages.push(error);
        }
        Ok(())
    }
}

impl Clone for NightlyChaosRunner {
    fn clone(&self) -> Self {
        Self {
            config: Arc::clone(&self.config),
            job_history: Arc::clone(&self.job_history),
            _event_sender: self._event_sender.clone(),
            is_running: Arc::clone(&self.is_running),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_nightly_chaos_runner() {
        let config = NightlyChaosConfig::default();
        let runner = NightlyChaosRunner::new(config);

        // Test event subscription
        let mut event_receiver = runner.subscribe_events();
        assert!(event_receiver.try_recv().is_err()); // No events yet

        // Test job history
        let history = runner.get_job_history().await;
        assert!(history.is_empty());
    }

    #[test]
    fn test_performance_summary() {
        let ml_results = vec![MLChaosResult {
            experiment_id: Uuid::new_v4(),
            model_type: super::ml_training_chaos::ModelType::TLOB,
            training_job_id: Some("test_job".to_string()),
            checkpoint_before_failure: None,
            checkpoint_after_recovery: None,
            model_accuracy_before: None,
            model_accuracy_after: None,
            training_loss_continuity: true,
            gpu_memory_recovery: None,
            performance_regression: None,
        }];

        let config = NightlyChaosConfig::default();
        let runner = NightlyChaosRunner::new(config);

        let summary = runner.calculate_performance_summary(&[], &ml_results);
        assert_eq!(summary.checkpoint_failures, 0);
    }
}