//! Chaos Engineering Module for Foxhunt HFT Trading System //! //! This module provides comprehensive chaos engineering capabilities specifically //! designed for high-frequency trading systems with sub-100ms recovery requirements. //! //! # Status: Disabled //! //! All submodules and tests are commented out because they require infrastructure //! that is not available in the standard test environment: //! //! - **ML gRPC service** (`ML_SERVICE_ENDPOINT`) must be running to inject training faults //! - **GPU infrastructure** for ML model chaos tests (checkpoint corruption, OOM simulation) //! - **Isolated environment** to safely inject network partitions, broker disconnects, and //! process crashes without affecting real trading or CI stability //! //! To run chaos tests, use the nightly chaos runner with `CHAOS_WEBHOOK_URL` configured //! and all dependent services deployed (see `tests/chaos/README.md` for setup instructions). // COMMENTED OUT - Need proper integration test harness with running services // pub mod chaos_cli; // pub mod chaos_framework; // pub mod examples; // pub mod ml_training_chaos; // pub mod nightly_chaos_runner; // COMMENTED OUT - All imports depend on disabled modules // use anyhow::Result; // use chrono; // use std::path::PathBuf; // use tracing::info; // use chaos_framework::ChaosOrchestrator; // use ml_training_chaos::{MLChaosConfig, MLChaosResult, MLTrainingChaosTests, ModelType}; // use nightly_chaos_runner::{NightlyChaosConfig, NightlyChaosRunner}; /* /// Initialize chaos engineering for the Foxhunt system (DISABLED) pub async fn initialize_foxhunt_chaos() -> Result { info!("Initializing Foxhunt chaos engineering framework"); // Configure chaos testing for HFT requirements let chaos_config = NightlyChaosConfig { enabled: true, schedule_time: chrono::NaiveTime::from_hms_opt(2, 0, 0).unwrap(), // 2 AM UTC timezone: "UTC".to_string(), max_duration_hours: 3, // Complete chaos testing within 3 hours notification_webhook: std::env::var("CHAOS_WEBHOOK_URL").ok(), report_storage_path: PathBuf::from("./chaos_reports"), ml_chaos_config: MLChaosConfig { ml_service_endpoint: std::env::var("ML_SERVICE_ENDPOINT") .unwrap_or_else(|_| "http://localhost:8080".to_string()), checkpoint_base_path: PathBuf::from("./ml_checkpoints"), model_types: vec![ ModelType::TLOB, // Ultra-low latency transformer ModelType::MAMBA2, // State space model ModelType::DQN, // Deep Q-learning ModelType::PPO, // Policy optimization ModelType::Liquid, // Liquid neural networks ModelType::TFT, // Temporal fusion transformer ], training_timeout_secs: 300, // 5 minutes max training time max_recovery_time_ms: 100, // HFT requirement: sub-100ms recovery gpu_memory_threshold_mb: 8192, // 8GB GPU memory threshold }, exclude_weekends: true, // Skip weekends for production safety retry_on_failure: true, max_retries: 2, }; let runner = NightlyChaosRunner::new(chaos_config); info!("Foxhunt chaos engineering framework initialized"); Ok(runner) } /// Quick chaos test for development/CI pub async fn run_quick_chaos_test() -> Result> { info!("Running quick chaos test for CI/development"); let ml_config = MLChaosConfig { ml_service_endpoint: "http://localhost:8080".to_string(), checkpoint_base_path: PathBuf::from("/tmp/test_checkpoints"), model_types: vec![ModelType::TLOB], // Just test TLOB for speed training_timeout_secs: 60, // 1 minute for quick test max_recovery_time_ms: 100, gpu_memory_threshold_mb: 2048, // Lower threshold for CI }; let ml_chaos = MLTrainingChaosTests::new(ml_config); // Run a subset of chaos tests let experiment_ids = ml_chaos.initialize_experiments().await?; let first_experiment = experiment_ids .into_iter() .next() .ok_or_else(|| anyhow::anyhow!("No experiments available"))?; // Execute just one experiment for quick testing let orchestrator = ChaosOrchestrator::new(1); let _result = orchestrator.execute_experiment(first_experiment).await?; // Return empty results for now (would implement actual quick test) Ok(vec![]) } */ // Tests disabled - require full service infrastructure /* #[cfg(test)] mod tests { use super::*; #[tokio::test] async fn test_chaos_initialization() { let runner = initialize_foxhunt_chaos().await; assert!(runner.is_ok()); } #[tokio::test] async fn test_quick_chaos_test() { // This would require actual ML service running // For now just test that the function exists let result = run_quick_chaos_test().await; // In CI without services running, this might fail, so we don't assert success println!("Quick chaos test result: {:?}", result); } } */