//! Failure Scenario Integration Tests for Foxhunt HFT System //! //! This module tests the system's resilience and recovery capabilities under various failure conditions: //! //! ## Failure Scenarios Covered: //! 1. **Kill Switch Activation**: Emergency shutdown procedures //! 2. **Trading Gate Guards**: Order processing gates under kill switch //! 3. **Scoped Kill Switches**: Symbol/Account/Strategy level blocks //! //! ## Recovery Validation: //! - System gracefully handles all failure modes //! - Recovery procedures restore full functionality //! - Kill switch activation is immediate //! - Trading gates prevent order submission when active #![warn(missing_docs)] #![warn(clippy::all)] #![allow(clippy::too_many_arguments)] #![allow(clippy::type_complexity)] #![allow(unused_crate_dependencies)] use std::sync::Arc; use std::time::{Duration, Instant}; use tracing::{info, warn}; // Core system imports - use full paths since types aren't re-exported from crate root use risk::AtomicKillSwitch; use risk::risk_types::KillSwitchScope; use risk::safety::KillSwitchConfig; use risk::safety::trading_gate::TradingGate; /// Failure scenario test configuration #[derive(Debug, Clone)] pub struct FailureTestConfig { /// Redis connection string for kill switch pub redis_url: String, /// Test timeout duration pub failure_timeout: Duration, /// Maximum acceptable activation time pub max_activation_time: Duration, } impl Default for FailureTestConfig { fn default() -> Self { Self { redis_url: std::env::var("TEST_REDIS_URL") .unwrap_or_else(|_| "redis://localhost:6379/2".to_string()), failure_timeout: Duration::from_secs(30), max_activation_time: Duration::from_millis(100), } } } /// Failure test harness for managing kill switch scenarios pub struct FailureTestHarness { config: FailureTestConfig, kill_switch: Arc, trading_gate: Arc, } impl FailureTestHarness { /// Create a new failure test harness pub async fn new() -> Result> { let config = FailureTestConfig::default(); info!("Initializing failure scenario test harness"); // Initialize kill switch with Redis backend let kill_switch_config = KillSwitchConfig::default(); let kill_switch = Arc::new( AtomicKillSwitch::new(kill_switch_config, config.redis_url.clone()).await? ); // Initialize trading gate let trading_gate = Arc::new(TradingGate::new(kill_switch.clone())); info!("Failure scenario test harness initialized successfully"); Ok(Self { config, kill_switch, trading_gate, }) } /// Test kill switch activation and emergency procedures pub async fn test_kill_switch_activation( &self, ) -> Result<(), Box> { info!("STARTING: Kill Switch Activation Test"); let start_time = Instant::now(); // Step 1: Verify normal operations (kill switch inactive) info!("Step 1: Verifying normal trading operations..."); let is_active = self.kill_switch.is_active().await?; if is_active { return Err("Kill switch should be inactive at start".into()); } // Verify trading gate allows orders let gate_result = self.trading_gate.pre_order_gate("AAPL", None); if gate_result.is_err() { return Err("Trading gate should allow orders when kill switch inactive".into()); } // Step 2: Activate kill switch info!("Step 2: Activating emergency kill switch..."); let activation_start = Instant::now(); self.kill_switch .activate( KillSwitchScope::Global, "Integration test emergency scenario".to_string(), "test_user".to_string(), false, ) .await?; let activation_time = activation_start.elapsed(); // Step 3: Verify kill switch is active info!("Step 3: Verifying kill switch activation..."); let is_active = self.kill_switch.is_active().await?; if !is_active { return Err("Kill switch should be active after activation".into()); } // Step 4: Verify all trading halts immediately info!("Step 4: Verifying immediate trading halt..."); let gate_result = self.trading_gate.pre_order_gate("AAPL", None); if gate_result.is_ok() { return Err("Trading gate should block orders when kill switch active".into()); } // Verify emergency check if self.trading_gate.emergency_check() { return Err("Emergency check should return false when kill switch active".into()); } // Step 5: Test recovery from kill switch info!("Step 5: Testing recovery from kill switch..."); self.kill_switch.deactivate( KillSwitchScope::Global, "Test completed".to_string(), ).await?; let is_active = self.kill_switch.is_active().await?; if is_active { return Err("Kill switch should be inactive after deactivation".into()); } // Verify trading gate allows orders again let gate_result = self.trading_gate.pre_order_gate("AAPL", None); if gate_result.is_err() { return Err("Trading gate should allow orders after kill switch deactivation".into()); } let test_duration = start_time.elapsed(); info!("KILL SWITCH ACTIVATION TEST COMPLETED"); info!(" Activation Time: {:?} (target: <{:?})", activation_time, self.config.max_activation_time); info!(" Total Duration: {:?}", test_duration); if activation_time > self.config.max_activation_time { warn!("Kill switch activation took longer than target: {:?} > {:?}", activation_time, self.config.max_activation_time); } Ok(()) } /// Test scoped kill switch activation (symbol-level) pub async fn test_scoped_kill_switch( &self, ) -> Result<(), Box> { info!("STARTING: Scoped Kill Switch Test"); // Step 1: Activate kill switch for specific symbol info!("Step 1: Activating kill switch for AAPL..."); self.kill_switch .activate( KillSwitchScope::Symbol("AAPL".to_string()), "Test symbol-level block".to_string(), "test_user".to_string(), false, ) .await?; // Step 2: Verify AAPL is blocked info!("Step 2: Verifying AAPL is blocked..."); let aapl_result = self.trading_gate.pre_order_gate("AAPL", None); if aapl_result.is_ok() { return Err("AAPL should be blocked by scoped kill switch".into()); } // Step 3: Verify MSFT is still allowed info!("Step 3: Verifying MSFT is still allowed..."); let msft_result = self.trading_gate.pre_order_gate("MSFT", None); if msft_result.is_err() { return Err("MSFT should not be affected by AAPL kill switch".into()); } // Step 4: Deactivate scoped kill switch info!("Step 4: Deactivating AAPL kill switch..."); self.kill_switch.deactivate( KillSwitchScope::Symbol("AAPL".to_string()), "Test completed".to_string(), ).await?; // Step 5: Verify AAPL is allowed again info!("Step 5: Verifying AAPL is allowed again..."); let aapl_result = self.trading_gate.pre_order_gate("AAPL", None); if aapl_result.is_err() { return Err("AAPL should be allowed after deactivation".into()); } info!("SCOPED KILL SWITCH TEST COMPLETED"); Ok(()) } /// Test trading gate performance under normal conditions pub async fn test_trading_gate_performance( &self, ) -> Result<(), Box> { info!("STARTING: Trading Gate Performance Test"); let num_checks = 10_000; let start_time = Instant::now(); for _ in 0..num_checks { let _ = self.trading_gate.pre_order_gate("AAPL", None); } let total_duration = start_time.elapsed(); let avg_latency_ns = total_duration.as_nanos() / num_checks; info!("TRADING GATE PERFORMANCE TEST COMPLETED"); info!(" Total Checks: {}", num_checks); info!(" Total Duration: {:?}", total_duration); info!(" Average Latency: {}ns", avg_latency_ns); // HFT compliance: each check should be sub-microsecond if avg_latency_ns > 1000 { warn!("Trading gate latency exceeds 1 microsecond target: {}ns", avg_latency_ns); } Ok(()) } /// Test multiple concurrent gate checks pub async fn test_concurrent_gate_checks( &self, ) -> Result<(), Box> { info!("STARTING: Concurrent Gate Checks Test"); let num_tasks = 100; let checks_per_task = 1000; let mut handles = Vec::new(); for task_id in 0..num_tasks { let gate = self.trading_gate.clone(); let handle = tokio::spawn(async move { for _ in 0..checks_per_task { let _ = gate.pre_order_gate("AAPL", None); } task_id }); handles.push(handle); } // Wait for all tasks to complete for handle in handles { handle.await?; } info!("CONCURRENT GATE CHECKS TEST COMPLETED"); info!(" Tasks: {}", num_tasks); info!(" Checks per Task: {}", checks_per_task); info!(" Total Checks: {}", num_tasks * checks_per_task); Ok(()) } /// Test batch symbol gate checking pub async fn test_batch_symbol_gate( &self, ) -> Result<(), Box> { info!("STARTING: Batch Symbol Gate Test"); let symbols = vec![ "AAPL".to_string(), "GOOGL".to_string(), "MSFT".to_string(), "TSLA".to_string(), ]; // Step 1: Test with no kill switches info!("Step 1: Testing batch gate with no kill switches..."); let allowed = self.trading_gate.batch_symbol_gate(&symbols)?; if allowed.len() != symbols.len() { return Err(format!("Expected {} allowed symbols, got {}", symbols.len(), allowed.len()).into()); } // Step 2: Activate kill switch for AAPL info!("Step 2: Activating kill switch for AAPL..."); self.kill_switch .activate( KillSwitchScope::Symbol("AAPL".to_string()), "Test batch filtering".to_string(), "test_user".to_string(), false, ) .await?; // Step 3: Test batch gate with one symbol blocked info!("Step 3: Testing batch gate with AAPL blocked..."); let allowed = self.trading_gate.batch_symbol_gate(&symbols)?; if allowed.len() != symbols.len() - 1 { return Err(format!("Expected {} allowed symbols, got {}", symbols.len() - 1, allowed.len()).into()); } if allowed.contains(&"AAPL".to_string()) { return Err("AAPL should not be in allowed symbols".into()); } // Step 4: Cleanup info!("Step 4: Cleaning up..."); self.kill_switch.deactivate( KillSwitchScope::Symbol("AAPL".to_string()), "Test completed".to_string(), ).await?; info!("BATCH SYMBOL GATE TEST COMPLETED"); Ok(()) } } // Integration test runner #[cfg(test)] mod tests { use super::*; #[tokio::test] async fn test_kill_switch_activation() { let harness = FailureTestHarness::new() .await .expect("Failed to initialize failure test harness"); harness .test_kill_switch_activation() .await .expect("Kill switch activation test failed"); } #[tokio::test] async fn test_scoped_kill_switch() { let harness = FailureTestHarness::new() .await .expect("Failed to initialize failure test harness"); harness .test_scoped_kill_switch() .await .expect("Scoped kill switch test failed"); } #[tokio::test] async fn test_trading_gate_performance() { let harness = FailureTestHarness::new() .await .expect("Failed to initialize failure test harness"); harness .test_trading_gate_performance() .await .expect("Trading gate performance test failed"); } #[tokio::test] async fn test_concurrent_gate_checks() { let harness = FailureTestHarness::new() .await .expect("Failed to initialize failure test harness"); harness .test_concurrent_gate_checks() .await .expect("Concurrent gate checks test failed"); } #[tokio::test] async fn test_batch_symbol_gate() { let harness = FailureTestHarness::new() .await .expect("Failed to initialize failure test harness"); harness .test_batch_symbol_gate() .await .expect("Batch symbol gate test failed"); } }