foxhunt/testing/integration/chaos/examples/usage_examples.rs

//! Chaos Engineering Usage Examples
//!
//! This file demonstrates how to use the Foxhunt chaos engineering framework
//! for testing system resilience and recovery capabilities.

use anyhow::Result;
use std::path::PathBuf;
use std::time::Duration;
use uuid::Uuid;

// Import our chaos engineering modules
use super::super::chaos_framework::{ChaosExperiment, ChaosOrchestrator, FailureType, Signal};
use super::super::ml_training_chaos::{MLChaosConfig, MLTrainingChaosTests, ModelType};
use crate::chaos::nightly_chaos_runner::{NightlyChaosConfig, NightlyChaosRunner, ChaosJobEvent};

/// Example 1: Simple ML Training Service Kill/Restart Test
///
/// This example demonstrates how to test the MLTrainingService's ability
/// to recover from process termination and resume training from checkpoints.
pub async fn example_ml_service_kill_test() -> Result<()> {
    println!("🧪 Example 1: ML Training Service Kill/Restart Test");

    // Create chaos orchestrator
    let orchestrator = ChaosOrchestrator::new(1);

    // Define the experiment
    let experiment = ChaosExperiment {
        id: Uuid::new_v4(),
        name: "MLTrainingService SIGTERM Recovery Test".to_string(),
        description: "Test ML service recovery from graceful termination".to_string(),
        target_service: "ml_training_service".to_string(),
        failure_type: FailureType::ProcessKill {
            signal: Signal::SIGTERM,
            delay_before_restart_ms: 2000, // 2 second delay
        },
        duration: Duration::from_secs(5), // Quick failure
        recovery_timeout: Duration::from_secs(30),
        max_recovery_time_ms: 100, // HFT requirement: sub-100ms
        enabled: true,
    };

    // Register and execute the experiment
    orchestrator.register_experiment(experiment.clone()).await?;
    let result = orchestrator.execute_experiment(experiment.id).await?;

    println!("📊 Result: {:?}", result.status);
    if let Some(recovery_time) = result.recovery_time_ms {
        println!("⏱️  Recovery time: {}ms", recovery_time);
        if recovery_time <= 100 {
            println!("✅ Recovery time meets HFT requirements");
        } else {
            println!("⚠️  Recovery time exceeds HFT requirements");
        }
    }

    println!(
        "🔧 Checkpoint integrity: {}",
        if result.checkpoint_integrity {
            "✅ Valid"
        } else {
            "❌ Corrupted"
        }
    );

    Ok(())
}

/// Example 2: Comprehensive ML Chaos Test Suite
///
/// This example runs the full ML chaos test suite across all supported models
/// with different failure scenarios and generates a comprehensive report.
pub async fn example_comprehensive_ml_chaos_suite() -> Result<()> {
    println!("🧪 Example 2: Comprehensive ML Chaos Test Suite");

    // Configure ML chaos testing
    let ml_config = MLChaosConfig {
        ml_service_endpoint: "http://localhost:8080".to_string(),
        checkpoint_base_path: PathBuf::from("/tmp/chaos_checkpoints"),
        model_types: vec![
            ModelType::TLOB,   // Ultra-low latency transformer
            ModelType::DQN,    // Deep Q-learning for strategy optimization
            ModelType::MAMBA2, // State space model for sequence modeling
        ],
        training_timeout_secs: 300,
        max_recovery_time_ms: 100,     // HFT requirement
        gpu_memory_threshold_mb: 4096, // 4GB for testing
    };

    // Create ML chaos test suite
    let ml_chaos = MLTrainingChaosTests::new(ml_config);

    println!("🚀 Starting ML chaos test suite...");
    let results = ml_chaos.run_ml_chaos_suite().await?;

    println!("📈 Test Results:");
    println!("   Total tests: {}", results.len());

    let successful = results
        .iter()
        .filter(|r| r.training_loss_continuity)
        .count();
    let failed = results.len() - successful;

    println!(
        "   Successful: {} ({}%)",
        successful,
        (successful * 100) / results.len()
    );
    println!(
        "   Failed: {} ({}%)",
        failed,
        (failed * 100) / results.len()
    );

    // Generate and display report
    let report = ml_chaos.generate_chaos_report(&results).await?;
    println!("\n📋 Detailed Report:");
    println!("{}", report);

    Ok(())
}

/// Example 3: Memory Pressure Test with Performance Monitoring
///
/// This example demonstrates how to test system behavior under memory pressure
/// while monitoring performance metrics and recovery times.
pub async fn example_memory_pressure_test() -> Result<()> {
    println!("🧪 Example 3: Memory Pressure Test with Performance Monitoring");

    let orchestrator = ChaosOrchestrator::new(1);

    // Create memory pressure experiment
    let experiment = ChaosExperiment {
        id: Uuid::new_v4(),
        name: "High Memory Pressure Test".to_string(),
        description: "Test system behavior under 4GB memory pressure".to_string(),
        target_service: "ml_training_service".to_string(),
        failure_type: FailureType::MemoryPressure {
            target_mb: 4096,    // 4GB pressure
            duration_ms: 30000, // 30 seconds
        },
        duration: Duration::from_secs(35),
        recovery_timeout: Duration::from_secs(60),
        max_recovery_time_ms: 200, // Relaxed for memory pressure
        enabled: true,
    };

    println!("💾 Applying 4GB memory pressure for 30 seconds...");

    orchestrator.register_experiment(experiment.clone()).await?;
    let result = orchestrator.execute_experiment(experiment.id).await?;

    println!("📊 Memory Pressure Test Results:");
    println!("   Status: {:?}", result.status);

    if let Some(recovery_time) = result.recovery_time_ms {
        println!("   Recovery time: {}ms", recovery_time);

        // Check performance regression
        if let Some(regression) = result.performance_regression {
            println!("   Performance impact:");
            println!(
                "     Before: {}ns P99 latency",
                regression.latency_p99_before_ns
            );
            println!(
                "     After: {}ns P99 latency",
                regression.latency_p99_after_ns
            );
            println!("     Regression: {:.1}%", regression.regression_percent);
        }
    }

    Ok(())
}

/// Example 4: Network Partition Resilience Test
///
/// This example tests the system's ability to handle network partitions
/// affecting critical services like PostgreSQL and Redis.
pub async fn example_network_partition_test() -> Result<()> {
    println!("🧪 Example 4: Network Partition Resilience Test");

    let orchestrator = ChaosOrchestrator::new(1);

    let experiment = ChaosExperiment {
        id: Uuid::new_v4(),
        name: "Database Network Partition Test".to_string(),
        description: "Test resilience to database connection failures".to_string(),
        target_service: "ml_training_service".to_string(),
        failure_type: FailureType::NetworkPartition {
            target_ports: vec![5432, 6379], // PostgreSQL, Redis
            duration_ms: 15000,             // 15 seconds
        },
        duration: Duration::from_secs(20),
        recovery_timeout: Duration::from_secs(45),
        max_recovery_time_ms: 100,
        enabled: true,
    };

    println!("🌐 Creating network partition affecting PostgreSQL and Redis...");

    orchestrator.register_experiment(experiment.clone()).await?;
    let result = orchestrator.execute_experiment(experiment.id).await?;

    println!("📡 Network Partition Test Results:");
    println!("   Status: {:?}", result.status);

    match result.status {
        super::super::chaos_framework::ChaosStatus::Succeeded => {
            println!("   ✅ System successfully handled network partition");
        }
        super::super::chaos_framework::ChaosStatus::RecoveryTimeout => {
            println!("   ⏰ Recovery took longer than expected");
        }
        super::super::chaos_framework::ChaosStatus::Failed => {
            println!("   ❌ System failed to recover from network partition");
        }
        _ => {
            println!("   ❓ Unexpected test status");
        }
    }

    Ok(())
}

/// Example 5: GPU Resource Exhaustion Test
///
/// This example tests ML model training behavior when GPU resources
/// are exhausted, simulating scenarios where multiple models compete
/// for limited GPU memory.
pub async fn example_gpu_exhaustion_test() -> Result<()> {
    println!("🧪 Example 5: GPU Resource Exhaustion Test");

    let orchestrator = ChaosOrchestrator::new(1);

    let experiment = ChaosExperiment {
        id: Uuid::new_v4(),
        name: "GPU Memory Exhaustion Test".to_string(),
        description: "Test ML training behavior under GPU memory pressure".to_string(),
        target_service: "ml_training_service".to_string(),
        failure_type: FailureType::GpuResourceExhaustion {
            memory_fill_percent: 95, // Fill 95% of GPU memory
            duration_ms: 25000,      // 25 seconds
        },
        duration: Duration::from_secs(30),
        recovery_timeout: Duration::from_secs(90), // GPU recovery can be slower
        max_recovery_time_ms: 200,                 // Relaxed for GPU operations
        enabled: true,
    };

    println!("🎮 Exhausting 95% of GPU memory for 25 seconds...");

    orchestrator.register_experiment(experiment.clone()).await?;
    let result = orchestrator.execute_experiment(experiment.id).await?;

    println!("🎯 GPU Exhaustion Test Results:");
    println!("   Status: {:?}", result.status);
    println!(
        "   Checkpoint integrity: {}",
        if result.checkpoint_integrity {
            "✅"
        } else {
            "❌"
        }
    );

    // GPU-specific analysis would check:
    // - Model training continuation after GPU memory cleared
    // - Checkpoint consistency during GPU pressure
    // - Training loss continuity
    // - GPU memory leak detection

    Ok(())
}

/// Example 6: Automated Nightly Chaos Testing
///
/// This example sets up automated nightly chaos testing with proper
/// scheduling, notification, and reporting.
pub async fn example_nightly_chaos_automation() -> Result<()> {
    println!("🧪 Example 6: Automated Nightly Chaos Testing Setup");

    // Configure nightly chaos testing
    let config = NightlyChaosConfig {
        enabled: true,
        schedule_time: chrono::NaiveTime::from_hms_opt(2, 0, 0).unwrap(), // 2 AM
        timezone: "UTC".to_string(),
        max_duration_hours: 3,
        notification_webhook: Some("https://hooks.slack.com/services/YOUR/WEBHOOK".to_string()),
        report_storage_path: PathBuf::from("./nightly_chaos_reports"),
        ml_chaos_config: MLChaosConfig {
            ml_service_endpoint: "http://localhost:8080".to_string(),
            checkpoint_base_path: PathBuf::from("/production/ml_checkpoints"),
            model_types: vec![
                ModelType::TLOB,   // Critical for order book analysis
                ModelType::DQN,    // Risk management
                ModelType::MAMBA2, // Market prediction
            ],
            training_timeout_secs: 600, // 10 minutes for production
            max_recovery_time_ms: 100,
            gpu_memory_threshold_mb: 16384, // 16GB production GPU
        },
        exclude_weekends: true, // Skip weekends for production safety
        retry_on_failure: true,
        max_retries: 2,
    };

    println!("📅 Setting up nightly chaos testing at 2:00 AM UTC...");
    println!("   Excluding weekends: {}", config.exclude_weekends);
    println!("   Max duration: {} hours", config.max_duration_hours);
    println!("   Report storage: {:?}", config.report_storage_path);

    // Create and configure the runner
    let runner = NightlyChaosRunner::new(config);

    // Subscribe to events for monitoring
    let mut event_receiver = runner.subscribe_events();

    // Start event monitoring in background
    let _event_handler = tokio::spawn(async move {
        while let Ok(event) = event_receiver.recv().await {
            match event {
                ChaosJobEvent::JobScheduled { id, scheduled_time } => {
                    println!("📅 Chaos job {} scheduled for {}", id, scheduled_time);
                }
                ChaosJobEvent::JobStarted { id } => {
                    println!("🚀 Chaos job {} started", id);
                }
                ChaosJobEvent::JobCompleted { id, summary } => {
                    println!("✅ Chaos job {} completed", id);
                    println!(
                        "   Average recovery time: {:.1}ms",
                        summary.average_recovery_time_ms
                    );
                    println!("   SLA violations: {}", summary.sla_violations);
                    println!("   Checkpoint failures: {}", summary.checkpoint_failures);
                }
                ChaosJobEvent::AlertTriggered {
                    message,
                    severity,
                } => {
                    println!("🚨 Alert ({:?}): {}", severity, message);
                }
                _ => {}
            }
        }
    });

    println!("🔄 Nightly chaos automation is configured and ready");
    println!("   Use `runner.start().await` to begin scheduled testing");

    // In production, you would call runner.start().await here
    // For this example, we just show the setup

    Ok(())
}

/// Example 7: CI/CD Integration - Quick Chaos Validation
///
/// This example shows how to integrate chaos testing into CI/CD pipelines
/// with quick validation tests that can run in a few minutes.
pub async fn example_ci_cd_quick_validation() -> Result<()> {
    println!("🧪 Example 7: CI/CD Quick Chaos Validation");

    // Quick config for CI/CD - shorter timeouts, fewer models
    let quick_ml_config = MLChaosConfig {
        ml_service_endpoint: "http://localhost:8080".to_string(),
        checkpoint_base_path: PathBuf::from("/tmp/ci_checkpoints"),
        model_types: vec![ModelType::TLOB], // Just test TLOB for speed
        training_timeout_secs: 60,          // 1 minute max
        max_recovery_time_ms: 100,
        gpu_memory_threshold_mb: 2048, // Lower for CI environment
    };

    println!("⚡ Running quick chaos validation for CI/CD...");
    println!("   Target: Sub-2 minute execution time");
    println!("   Models: TLOB only (fastest)");
    println!("   Recovery requirement: < 100ms");

    let ml_chaos = MLTrainingChaosTests::new(quick_ml_config);

    // Run a single representative test
    let orchestrator = ChaosOrchestrator::new(1);

    let quick_experiment = ChaosExperiment {
        id: Uuid::new_v4(),
        name: "CI Quick Recovery Test".to_string(),
        description: "Fast recovery test for CI pipeline".to_string(),
        target_service: "ml_training_service".to_string(),
        failure_type: FailureType::ProcessKill {
            signal: Signal::SIGTERM,
            delay_before_restart_ms: 1000, // Quick restart
        },
        duration: Duration::from_secs(2), // Very quick failure
        recovery_timeout: Duration::from_secs(15), // Quick recovery
        max_recovery_time_ms: 100,
        enabled: true,
    };

    let start_time = std::time::Instant::now();

    orchestrator
        .register_experiment(quick_experiment.clone())
        .await?;
    let result = orchestrator.execute_experiment(quick_experiment.id).await?;

    let total_time = start_time.elapsed();

    println!("⏱️  Total execution time: {:.1}s", total_time.as_secs_f64());
    println!("📊 Quick validation result: {:?}", result.status);

    match result.status {
        super::super::chaos_framework::ChaosStatus::Succeeded => {
            println!("✅ CI/CD chaos validation PASSED");
            std::process::exit(0);
        }
        _ => {
            println!("❌ CI/CD chaos validation FAILED");
            if let Some(recovery_time) = result.recovery_time_ms {
                println!("   Recovery time: {}ms", recovery_time);
            }
            println!("   Checkpoint integrity: {}", result.checkpoint_integrity);
            std::process::exit(1);
        }
    }
}

/// Run all examples
pub async fn run_all_examples() -> Result<()> {
    println!("🚀 Running Foxhunt Chaos Engineering Examples\n");

    // Example 1: Basic ML service kill test
    example_ml_service_kill_test().await?;
    println!();

    // Example 2: Comprehensive test suite
    example_comprehensive_ml_chaos_suite().await?;
    println!();

    // Example 3: Memory pressure test
    example_memory_pressure_test().await?;
    println!();

    // Example 4: Network partition test
    example_network_partition_test().await?;
    println!();

    // Example 5: GPU exhaustion test
    example_gpu_exhaustion_test().await?;
    println!();

    // Example 6: Nightly automation setup
    example_nightly_chaos_automation().await?;
    println!();

    // Example 7: CI/CD integration
    // example_ci_cd_quick_validation().await?; // Skip in demo to avoid exit

    println!("🎉 All chaos engineering examples completed successfully!");

    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_example_runs() {
        // Test that examples can be set up without actual service calls
        let ml_config = MLChaosConfig {
            ml_service_endpoint: "http://localhost:8080".to_string(),
            checkpoint_base_path: PathBuf::from("/tmp/test"),
            model_types: vec![ModelType::TLOB],
            training_timeout_secs: 60,
            max_recovery_time_ms: 100,
            gpu_memory_threshold_mb: 2048,
        };

        let _ml_chaos = MLTrainingChaosTests::new(ml_config);
        assert!(true); // Basic creation test
    }
}