//! Chaos Engineering Usage Examples //! //! This file demonstrates how to use the Foxhunt chaos engineering framework //! for testing system resilience and recovery capabilities. use anyhow::Result; use std::path::PathBuf; use std::time::Duration; use uuid::Uuid; // Import our chaos engineering modules use super::super::chaos_framework::{ChaosExperiment, ChaosOrchestrator, FailureType, Signal}; use super::super::ml_training_chaos::{MLChaosConfig, MLTrainingChaosTests, ModelType}; use crate::chaos::nightly_chaos_runner::{NightlyChaosConfig, NightlyChaosRunner, ChaosJobEvent}; /// Example 1: Simple ML Training Service Kill/Restart Test /// /// This example demonstrates how to test the MLTrainingService's ability /// to recover from process termination and resume training from checkpoints. pub async fn example_ml_service_kill_test() -> Result<()> { println!("๐Ÿงช Example 1: ML Training Service Kill/Restart Test"); // Create chaos orchestrator let orchestrator = ChaosOrchestrator::new(1); // Define the experiment let experiment = ChaosExperiment { id: Uuid::new_v4(), name: "MLTrainingService SIGTERM Recovery Test".to_string(), description: "Test ML service recovery from graceful termination".to_string(), target_service: "ml_training_service".to_string(), failure_type: FailureType::ProcessKill { signal: Signal::SIGTERM, delay_before_restart_ms: 2000, // 2 second delay }, duration: Duration::from_secs(5), // Quick failure recovery_timeout: Duration::from_secs(30), max_recovery_time_ms: 100, // HFT requirement: sub-100ms enabled: true, }; // Register and execute the experiment orchestrator.register_experiment(experiment.clone()).await?; let result = orchestrator.execute_experiment(experiment.id).await?; println!("๐Ÿ“Š Result: {:?}", result.status); if let Some(recovery_time) = result.recovery_time_ms { println!("โฑ๏ธ Recovery time: {}ms", recovery_time); if recovery_time <= 100 { println!("โœ… Recovery time meets HFT requirements"); } else { println!("โš ๏ธ Recovery time exceeds HFT requirements"); } } println!( "๐Ÿ”ง Checkpoint integrity: {}", if result.checkpoint_integrity { "โœ… Valid" } else { "โŒ Corrupted" } ); Ok(()) } /// Example 2: Comprehensive ML Chaos Test Suite /// /// This example runs the full ML chaos test suite across all supported models /// with different failure scenarios and generates a comprehensive report. pub async fn example_comprehensive_ml_chaos_suite() -> Result<()> { println!("๐Ÿงช Example 2: Comprehensive ML Chaos Test Suite"); // Configure ML chaos testing let ml_config = MLChaosConfig { ml_service_endpoint: "http://localhost:8080".to_string(), checkpoint_base_path: PathBuf::from("/tmp/chaos_checkpoints"), model_types: vec![ ModelType::TLOB, // Ultra-low latency transformer ModelType::DQN, // Deep Q-learning for strategy optimization ModelType::MAMBA2, // State space model for sequence modeling ], training_timeout_secs: 300, max_recovery_time_ms: 100, // HFT requirement gpu_memory_threshold_mb: 4096, // 4GB for testing }; // Create ML chaos test suite let ml_chaos = MLTrainingChaosTests::new(ml_config); println!("๐Ÿš€ Starting ML chaos test suite..."); let results = ml_chaos.run_ml_chaos_suite().await?; println!("๐Ÿ“ˆ Test Results:"); println!(" Total tests: {}", results.len()); let successful = results .iter() .filter(|r| r.training_loss_continuity) .count(); let failed = results.len() - successful; println!( " Successful: {} ({}%)", successful, (successful * 100) / results.len() ); println!( " Failed: {} ({}%)", failed, (failed * 100) / results.len() ); // Generate and display report let report = ml_chaos.generate_chaos_report(&results).await?; println!("\n๐Ÿ“‹ Detailed Report:"); println!("{}", report); Ok(()) } /// Example 3: Memory Pressure Test with Performance Monitoring /// /// This example demonstrates how to test system behavior under memory pressure /// while monitoring performance metrics and recovery times. pub async fn example_memory_pressure_test() -> Result<()> { println!("๐Ÿงช Example 3: Memory Pressure Test with Performance Monitoring"); let orchestrator = ChaosOrchestrator::new(1); // Create memory pressure experiment let experiment = ChaosExperiment { id: Uuid::new_v4(), name: "High Memory Pressure Test".to_string(), description: "Test system behavior under 4GB memory pressure".to_string(), target_service: "ml_training_service".to_string(), failure_type: FailureType::MemoryPressure { target_mb: 4096, // 4GB pressure duration_ms: 30000, // 30 seconds }, duration: Duration::from_secs(35), recovery_timeout: Duration::from_secs(60), max_recovery_time_ms: 200, // Relaxed for memory pressure enabled: true, }; println!("๐Ÿ’พ Applying 4GB memory pressure for 30 seconds..."); orchestrator.register_experiment(experiment.clone()).await?; let result = orchestrator.execute_experiment(experiment.id).await?; println!("๐Ÿ“Š Memory Pressure Test Results:"); println!(" Status: {:?}", result.status); if let Some(recovery_time) = result.recovery_time_ms { println!(" Recovery time: {}ms", recovery_time); // Check performance regression if let Some(regression) = result.performance_regression { println!(" Performance impact:"); println!( " Before: {}ns P99 latency", regression.latency_p99_before_ns ); println!( " After: {}ns P99 latency", regression.latency_p99_after_ns ); println!(" Regression: {:.1}%", regression.regression_percent); } } Ok(()) } /// Example 4: Network Partition Resilience Test /// /// This example tests the system's ability to handle network partitions /// affecting critical services like PostgreSQL and Redis. pub async fn example_network_partition_test() -> Result<()> { println!("๐Ÿงช Example 4: Network Partition Resilience Test"); let orchestrator = ChaosOrchestrator::new(1); let experiment = ChaosExperiment { id: Uuid::new_v4(), name: "Database Network Partition Test".to_string(), description: "Test resilience to database connection failures".to_string(), target_service: "ml_training_service".to_string(), failure_type: FailureType::NetworkPartition { target_ports: vec![5432, 6379], // PostgreSQL, Redis duration_ms: 15000, // 15 seconds }, duration: Duration::from_secs(20), recovery_timeout: Duration::from_secs(45), max_recovery_time_ms: 100, enabled: true, }; println!("๐ŸŒ Creating network partition affecting PostgreSQL and Redis..."); orchestrator.register_experiment(experiment.clone()).await?; let result = orchestrator.execute_experiment(experiment.id).await?; println!("๐Ÿ“ก Network Partition Test Results:"); println!(" Status: {:?}", result.status); match result.status { super::super::chaos_framework::ChaosStatus::Succeeded => { println!(" โœ… System successfully handled network partition"); } super::super::chaos_framework::ChaosStatus::RecoveryTimeout => { println!(" โฐ Recovery took longer than expected"); } super::super::chaos_framework::ChaosStatus::Failed => { println!(" โŒ System failed to recover from network partition"); } _ => { println!(" โ“ Unexpected test status"); } } Ok(()) } /// Example 5: GPU Resource Exhaustion Test /// /// This example tests ML model training behavior when GPU resources /// are exhausted, simulating scenarios where multiple models compete /// for limited GPU memory. pub async fn example_gpu_exhaustion_test() -> Result<()> { println!("๐Ÿงช Example 5: GPU Resource Exhaustion Test"); let orchestrator = ChaosOrchestrator::new(1); let experiment = ChaosExperiment { id: Uuid::new_v4(), name: "GPU Memory Exhaustion Test".to_string(), description: "Test ML training behavior under GPU memory pressure".to_string(), target_service: "ml_training_service".to_string(), failure_type: FailureType::GpuResourceExhaustion { memory_fill_percent: 95, // Fill 95% of GPU memory duration_ms: 25000, // 25 seconds }, duration: Duration::from_secs(30), recovery_timeout: Duration::from_secs(90), // GPU recovery can be slower max_recovery_time_ms: 200, // Relaxed for GPU operations enabled: true, }; println!("๐ŸŽฎ Exhausting 95% of GPU memory for 25 seconds..."); orchestrator.register_experiment(experiment.clone()).await?; let result = orchestrator.execute_experiment(experiment.id).await?; println!("๐ŸŽฏ GPU Exhaustion Test Results:"); println!(" Status: {:?}", result.status); println!( " Checkpoint integrity: {}", if result.checkpoint_integrity { "โœ…" } else { "โŒ" } ); // GPU-specific analysis would check: // - Model training continuation after GPU memory cleared // - Checkpoint consistency during GPU pressure // - Training loss continuity // - GPU memory leak detection Ok(()) } /// Example 6: Automated Nightly Chaos Testing /// /// This example sets up automated nightly chaos testing with proper /// scheduling, notification, and reporting. pub async fn example_nightly_chaos_automation() -> Result<()> { println!("๐Ÿงช Example 6: Automated Nightly Chaos Testing Setup"); // Configure nightly chaos testing let config = NightlyChaosConfig { enabled: true, schedule_time: chrono::NaiveTime::from_hms_opt(2, 0, 0).unwrap(), // 2 AM timezone: "UTC".to_string(), max_duration_hours: 3, notification_webhook: Some("https://hooks.slack.com/services/YOUR/WEBHOOK".to_string()), report_storage_path: PathBuf::from("./nightly_chaos_reports"), ml_chaos_config: MLChaosConfig { ml_service_endpoint: "http://localhost:8080".to_string(), checkpoint_base_path: PathBuf::from("/production/ml_checkpoints"), model_types: vec![ ModelType::TLOB, // Critical for order book analysis ModelType::DQN, // Risk management ModelType::MAMBA2, // Market prediction ], training_timeout_secs: 600, // 10 minutes for production max_recovery_time_ms: 100, gpu_memory_threshold_mb: 16384, // 16GB production GPU }, exclude_weekends: true, // Skip weekends for production safety retry_on_failure: true, max_retries: 2, }; println!("๐Ÿ“… Setting up nightly chaos testing at 2:00 AM UTC..."); println!(" Excluding weekends: {}", config.exclude_weekends); println!(" Max duration: {} hours", config.max_duration_hours); println!(" Report storage: {:?}", config.report_storage_path); // Create and configure the runner let runner = NightlyChaosRunner::new(config); // Subscribe to events for monitoring let mut event_receiver = runner.subscribe_events(); // Start event monitoring in background let _event_handler = tokio::spawn(async move { while let Ok(event) = event_receiver.recv().await { match event { ChaosJobEvent::JobScheduled { id, scheduled_time } => { println!("๐Ÿ“… Chaos job {} scheduled for {}", id, scheduled_time); } ChaosJobEvent::JobStarted { id } => { println!("๐Ÿš€ Chaos job {} started", id); } ChaosJobEvent::JobCompleted { id, summary } => { println!("โœ… Chaos job {} completed", id); println!( " Average recovery time: {:.1}ms", summary.average_recovery_time_ms ); println!(" SLA violations: {}", summary.sla_violations); println!(" Checkpoint failures: {}", summary.checkpoint_failures); } ChaosJobEvent::AlertTriggered { message, severity, } => { println!("๐Ÿšจ Alert ({:?}): {}", severity, message); } _ => {} } } }); println!("๐Ÿ”„ Nightly chaos automation is configured and ready"); println!(" Use `runner.start().await` to begin scheduled testing"); // In production, you would call runner.start().await here // For this example, we just show the setup Ok(()) } /// Example 7: CI/CD Integration - Quick Chaos Validation /// /// This example shows how to integrate chaos testing into CI/CD pipelines /// with quick validation tests that can run in a few minutes. pub async fn example_ci_cd_quick_validation() -> Result<()> { println!("๐Ÿงช Example 7: CI/CD Quick Chaos Validation"); // Quick config for CI/CD - shorter timeouts, fewer models let quick_ml_config = MLChaosConfig { ml_service_endpoint: "http://localhost:8080".to_string(), checkpoint_base_path: PathBuf::from("/tmp/ci_checkpoints"), model_types: vec![ModelType::TLOB], // Just test TLOB for speed training_timeout_secs: 60, // 1 minute max max_recovery_time_ms: 100, gpu_memory_threshold_mb: 2048, // Lower for CI environment }; println!("โšก Running quick chaos validation for CI/CD..."); println!(" Target: Sub-2 minute execution time"); println!(" Models: TLOB only (fastest)"); println!(" Recovery requirement: < 100ms"); let ml_chaos = MLTrainingChaosTests::new(quick_ml_config); // Run a single representative test let orchestrator = ChaosOrchestrator::new(1); let quick_experiment = ChaosExperiment { id: Uuid::new_v4(), name: "CI Quick Recovery Test".to_string(), description: "Fast recovery test for CI pipeline".to_string(), target_service: "ml_training_service".to_string(), failure_type: FailureType::ProcessKill { signal: Signal::SIGTERM, delay_before_restart_ms: 1000, // Quick restart }, duration: Duration::from_secs(2), // Very quick failure recovery_timeout: Duration::from_secs(15), // Quick recovery max_recovery_time_ms: 100, enabled: true, }; let start_time = std::time::Instant::now(); orchestrator .register_experiment(quick_experiment.clone()) .await?; let result = orchestrator.execute_experiment(quick_experiment.id).await?; let total_time = start_time.elapsed(); println!("โฑ๏ธ Total execution time: {:.1}s", total_time.as_secs_f64()); println!("๐Ÿ“Š Quick validation result: {:?}", result.status); match result.status { super::super::chaos_framework::ChaosStatus::Succeeded => { println!("โœ… CI/CD chaos validation PASSED"); std::process::exit(0); } _ => { println!("โŒ CI/CD chaos validation FAILED"); if let Some(recovery_time) = result.recovery_time_ms { println!(" Recovery time: {}ms", recovery_time); } println!(" Checkpoint integrity: {}", result.checkpoint_integrity); std::process::exit(1); } } } /// Run all examples pub async fn run_all_examples() -> Result<()> { println!("๐Ÿš€ Running Foxhunt Chaos Engineering Examples\n"); // Example 1: Basic ML service kill test example_ml_service_kill_test().await?; println!(); // Example 2: Comprehensive test suite example_comprehensive_ml_chaos_suite().await?; println!(); // Example 3: Memory pressure test example_memory_pressure_test().await?; println!(); // Example 4: Network partition test example_network_partition_test().await?; println!(); // Example 5: GPU exhaustion test example_gpu_exhaustion_test().await?; println!(); // Example 6: Nightly automation setup example_nightly_chaos_automation().await?; println!(); // Example 7: CI/CD integration // example_ci_cd_quick_validation().await?; // Skip in demo to avoid exit println!("๐ŸŽ‰ All chaos engineering examples completed successfully!"); Ok(()) } #[cfg(test)] mod tests { use super::*; #[tokio::test] async fn test_example_runs() { // Test that examples can be set up without actual service calls let ml_config = MLChaosConfig { ml_service_endpoint: "http://localhost:8080".to_string(), checkpoint_base_path: PathBuf::from("/tmp/test"), model_types: vec![ModelType::TLOB], training_timeout_secs: 60, max_recovery_time_ms: 100, gpu_memory_threshold_mb: 2048, }; let _ml_chaos = MLTrainingChaosTests::new(ml_config); assert!(true); // Basic creation test } }