//! Safety Tests for Foxhunt HFT Trading System //! //! This module tests all safety mechanisms that protect the trading system from //! catastrophic failures. These tests ensure that emergency controls work correctly //! under various failure scenarios. //! //! # Safety Test Coverage //! //! - **Emergency Kill Switch** (Global, per-strategy, per-symbol) //! - **Circuit Breakers** (Dynamic thresholds, portfolio protection) //! - **Position Limiters** (Hard limits, concentration risk) //! - **Drawdown Protection** (Real-time monitoring, automatic stops) //! - **Risk Escalation** (Alert systems, emergency response) //! - **Financial Safety** (Overflow protection, precision handling) //! - **Memory Safety** (Bounds checking, resource limits) //! - **Concurrent Safety** (Thread safety, atomic operations) //! //! # Test Philosophy //! //! Safety tests are designed to validate that protective mechanisms work even //! under extreme conditions. They test both normal operation and edge cases //! that could lead to system failure or financial loss. use anyhow::Result; use std::time::{Duration, Instant}; use std::sync::{Arc, atomic::{AtomicBool, AtomicU64, Ordering}}; use std::collections::HashMap; use tokio::time::timeout; // Import unified types // Import risk and safety systems // use risk::prelude::*; // REMOVED - prelude does not exist // Import common test utilities use crate::common::{*, test_config::*, test_utils::*, assertions::*}; /// Safety test configuration #[derive(Debug, Clone)] struct SafetyTestConfig { /// Test timeout for safety operations timeout_seconds: u64, /// Enable Redis-based kill switch testing enable_redis_tests: bool, /// Emergency response email (for testing) test_email: String, /// Maximum allowed safety check latency max_safety_latency_us: u64, /// Position limits for testing test_position_limits: TestPositionLimits, /// Drawdown limits for testing test_drawdown_limits: TestDrawdownLimits, } #[derive(Debug, Clone)] struct TestPositionLimits { max_position_per_symbol: f64, max_total_exposure: f64, max_concentration_ratio: f64, max_order_size: f64, } #[derive(Debug, Clone)] struct TestDrawdownLimits { max_daily_loss: f64, max_drawdown: f64, consecutive_loss_limit: u32, loss_check_interval_ms: u64, } impl Default for SafetyTestConfig { fn default() -> Self { Self { timeout_seconds: 30, enable_redis_tests: false, // Disabled by default for CI/CD test_email: "test@foxhunt.local".to_string(), max_safety_latency_us: 10, // 10μs for safety operations test_position_limits: TestPositionLimits { max_position_per_symbol: 10000.0, max_total_exposure: 50000.0, max_concentration_ratio: 0.1, // 10% max per symbol max_order_size: 5000.0, }, test_drawdown_limits: TestDrawdownLimits { max_daily_loss: 1000.0, max_drawdown: 2000.0, consecutive_loss_limit: 3, loss_check_interval_ms: 100, }, } } } /// Test position for safety validation #[derive(Debug, Clone)] struct TestPosition { symbol: Symbol, quantity: Quantity, avg_price: Price, current_price: Price, unrealized_pnl: Price, timestamp: HftTimestamp, } impl TestPosition { fn new(symbol: &str, quantity: f64, avg_price: f64, current_price: f64) -> Result { let unrealized_pnl = Price::from_f64((current_price - avg_price) * quantity)?; Ok(Self { symbol: Symbol::from(symbol), quantity: Quantity::from_f64(quantity)?, avg_price: Price::from_f64(avg_price)?, current_price: Price::from_f64(current_price)?, unrealized_pnl, timestamp: HftTimestamp::now()?, }) } fn market_value(&self) -> Price { Price::from_f64(self.quantity.to_f64() * self.current_price.to_f64()) .unwrap_or(Price::ZERO) } fn is_profitable(&self) -> bool { self.unrealized_pnl.to_f64() > 0.0 } } /// Emergency event for testing #[derive(Debug, Clone)] struct EmergencyEvent { event_type: EmergencyType, severity: RiskSeverity, message: String, timestamp: HftTimestamp, triggered_by: String, automatic_response: bool, } #[derive(Debug, Clone, PartialEq)] enum EmergencyType { PositionLimit, DrawdownLimit, SystemFailure, NetworkFailure, MarketDisruption, RiskViolation, } impl EmergencyEvent { fn new(event_type: EmergencyType, severity: RiskSeverity, message: &str) -> Result { Ok(Self { event_type, severity, message: message.to_string(), timestamp: HftTimestamp::now()?, triggered_by: "safety_test".to_string(), automatic_response: true, }) } fn requires_immediate_action(&self) -> bool { matches!(self.severity, RiskSeverity::Critical | RiskSeverity::High) } } /// Safety test suite struct SafetyTestSuite { config: SafetyTestConfig, kill_switch: Option, position_limiter: Option, drawdown_monitor: Option, safety_coordinator: Option, emergency_events: Arc>>, kill_switch_triggered: Arc, total_safety_checks: Arc, } impl SafetyTestSuite { fn new() -> Self { setup_test_tracing(); Self { config: SafetyTestConfig::default(), kill_switch: None, position_limiter: None, drawdown_monitor: None, safety_coordinator: None, emergency_events: Arc::new(std::sync::Mutex::new(Vec::new())), kill_switch_triggered: Arc::new(AtomicBool::new(false)), total_safety_checks: Arc::new(AtomicU64::new(0)), } } async fn setup(&mut self) -> Result<()> { // Initialize safety configuration let safety_config = SafetyConfig { enabled: true, kill_switch: KillSwitchConfig { enabled: true, global_channel: "test:kill_switch:global".to_string(), strategy_channel_prefix: "test:kill_switch:strategy".to_string(), symbol_channel_prefix: "test:kill_switch:symbol".to_string(), auto_recovery_enabled: false, // Manual for testing auto_recovery_delay: Duration::from_secs(60), }, position_limits: PositionLimiterConfig { enabled: true, cache_ttl: Duration::from_secs(10), rpc_check_threshold_percent: 0.8, max_position_per_symbol: self.config.test_position_limits.max_position_per_symbol, max_order_value: self.config.test_position_limits.max_order_size, max_daily_loss: self.config.test_drawdown_limits.max_daily_loss, }, emergency_response: EmergencyResponseConfig { enabled: true, loss_check_interval: Duration::from_millis(self.config.test_drawdown_limits.loss_check_interval_ms), position_check_interval: Duration::from_millis(50), max_consecutive_violations: self.config.test_drawdown_limits.consecutive_loss_limit, emergency_contacts: vec![self.config.test_email.clone()], max_daily_loss: Price::from_f64(self.config.test_drawdown_limits.max_daily_loss)?, max_drawdown: Price::from_f64(self.config.test_drawdown_limits.max_drawdown)?, }, redis_url: "redis://localhost:6379".to_string(), safety_check_timeout: Duration::from_micros(self.config.max_safety_latency_us), }; // Initialize safety components self.kill_switch = Some(AtomicKillSwitch::new(safety_config.kill_switch.clone())?); self.position_limiter = Some(HybridPositionLimiter::new(safety_config.position_limits)?); self.drawdown_monitor = Some(DrawdownMonitor::new( safety_config.emergency_response.max_daily_loss, safety_config.emergency_response.max_drawdown, )?); self.safety_coordinator = Some(SafetyCoordinator::new(safety_config).await?); Ok(()) } /// Record emergency event fn record_emergency_event(&self, event: EmergencyEvent) { if let Ok(mut events) = self.emergency_events.lock() { events.push(event); } } /// Get emergency event count by type fn get_emergency_count(&self, event_type: EmergencyType) -> usize { if let Ok(events) = self.emergency_events.lock() { events.iter().filter(|e| e.event_type == event_type).count() } else { 0 } } /// Simulate position that violates limits fn create_violating_position(&self) -> Result { TestPosition::new( "VIOLATION_TEST", self.config.test_position_limits.max_position_per_symbol * 2.0, // 2x limit 50000.0, 51000.0, ) } /// Simulate safe position within limits fn create_safe_position(&self) -> Result { TestPosition::new( "SAFE_TEST", self.config.test_position_limits.max_position_per_symbol * 0.5, // 50% of limit 50000.0, 50100.0, ) } /// Test kill switch functionality async fn test_kill_switch_activation(&mut self) -> Result<()> { if let Some(ref mut kill_switch) = self.kill_switch { let start_time = Instant::now(); // Test global kill switch kill_switch.trigger_global_kill().await?; // Verify kill switch state assert!(kill_switch.is_killed().await?, "Kill switch should be activated"); // Record that kill switch was triggered self.kill_switch_triggered.store(true, Ordering::SeqCst); // Validate activation latency assert_hft_latency(start_time.elapsed(), self.config.max_safety_latency_us); // Test recovery (if enabled) if kill_switch.can_recover().await? { kill_switch.recover_global().await?; assert!(!kill_switch.is_killed().await?, "Kill switch should be recovered"); } } Ok(()) } /// Test position limit enforcement async fn test_position_limits(&self) -> Result<()> { if let Some(ref position_limiter) = self.position_limiter { // Test safe position let safe_position = self.create_safe_position()?; let safe_check = position_limiter.check_position_limit( &safe_position.symbol, safe_position.quantity, safe_position.current_price, ).await; // Should not violate limits match safe_check { Ok(_) => {}, // Position accepted Err(_) => {}, // May be rejected due to other factors - that's OK } // Test violating position let violating_position = self.create_violating_position()?; let violation_check = position_limiter.check_position_limit( &violating_position.symbol, violating_position.quantity, violating_position.current_price, ).await; // Should be rejected or trigger safety mechanisms match violation_check { Ok(_) => tracing::warn!("Expected position limit violation but was allowed"), Err(_) => { // Position correctly rejected self.record_emergency_event(EmergencyEvent::new( EmergencyType::PositionLimit, RiskSeverity::High, "Position limit violation detected", )?); } } } Ok(()) } /// Test drawdown protection async fn test_drawdown_protection(&self) -> Result<()> { if let Some(ref drawdown_monitor) = self.drawdown_monitor { let start_time = Instant::now(); // Simulate small loss (within limits) let small_loss = Price::from_f64(-100.0)?; drawdown_monitor.record_pnl(small_loss).await?; // Should not trigger protection assert!(!drawdown_monitor.is_limit_breached().await?, "Small loss should not trigger drawdown protection"); // Simulate large loss (exceeding limits) let large_loss = Price::from_f64(-self.config.test_drawdown_limits.max_daily_loss * 1.5)?; drawdown_monitor.record_pnl(large_loss).await?; // Should trigger protection assert!(drawdown_monitor.is_limit_breached().await?, "Large loss should trigger drawdown protection"); // Record emergency event self.record_emergency_event(EmergencyEvent::new( EmergencyType::DrawdownLimit, RiskSeverity::Critical, "Drawdown limit exceeded", )?); // Validate response latency assert_hft_latency(start_time.elapsed(), self.config.max_safety_latency_us); } Ok(()) } /// Test emergency response coordination async fn test_emergency_response(&self) -> Result<()> { if let Some(ref safety_coordinator) = self.safety_coordinator { // Create emergency scenario let emergency = EmergencyEvent::new( EmergencyType::SystemFailure, RiskSeverity::Critical, "Critical system failure detected", )?; let start_time = Instant::now(); // Trigger emergency response let response = safety_coordinator.handle_emergency(&emergency.message).await; // Validate response match response { Ok(_) => { tracing::info!("Emergency response completed successfully"); } Err(e) => { tracing::warn!("Emergency response failed: {}", e); // Failure to respond is itself a critical issue } } // Validate response latency (should be immediate) assert_hft_latency(start_time.elapsed(), self.config.max_safety_latency_us); self.record_emergency_event(emergency); } Ok(()) } /// Increment safety check counter fn increment_safety_checks(&self) { self.total_safety_checks.fetch_add(1, Ordering::SeqCst); } } // ========== SAFETY MECHANISM TESTS ========== #[tokio::test] async fn test_emergency_kill_switch_activation() -> Result<()> { let mut test_suite = SafetyTestSuite::new(); test_suite.setup().await?; // Test global kill switch test_suite.test_kill_switch_activation().await?; // Verify kill switch was triggered assert!(test_suite.kill_switch_triggered.load(Ordering::SeqCst), "Kill switch should have been triggered"); Ok(()) } #[tokio::test] async fn test_position_limit_enforcement() -> Result<()> { let mut test_suite = SafetyTestSuite::new(); test_suite.setup().await?; // Test position limits test_suite.test_position_limits().await?; // Check if any position limit violations were recorded let violation_count = test_suite.get_emergency_count(EmergencyType::PositionLimit); tracing::info!("Position limit violations detected: {}", violation_count); Ok(()) } #[tokio::test] async fn test_drawdown_protection_mechanisms() -> Result<()> { let mut test_suite = SafetyTestSuite::new(); test_suite.setup().await?; // Test drawdown protection test_suite.test_drawdown_protection().await?; // Verify drawdown events were recorded let drawdown_count = test_suite.get_emergency_count(EmergencyType::DrawdownLimit); assert!(drawdown_count > 0, "Drawdown protection should have been triggered"); Ok(()) } #[tokio::test] async fn test_circuit_breaker_functionality() -> Result<()> { let mut test_suite = SafetyTestSuite::new(); test_suite.setup().await?; // Test circuit breaker configuration let circuit_config = CircuitBreakerConfig { enabled: true, failure_threshold: 3, timeout_duration: Duration::from_millis(100), half_open_timeout: Duration::from_secs(10), }; // Simulate multiple failures to trigger circuit breaker let mut failure_count = 0; for i in 0..5 { // Simulate operation that might fail let operation_result = simulate_risky_operation(i).await; if operation_result.is_err() { failure_count += 1; test_suite.increment_safety_checks(); } // Circuit breaker should open after threshold failures if failure_count >= circuit_config.failure_threshold { tracing::info!("Circuit breaker should be open after {} failures", failure_count); break; } } assert!(failure_count > 0, "Should have recorded some failures for circuit breaker testing"); Ok(()) } /// Simulate an operation that might fail (for circuit breaker testing) async fn simulate_risky_operation(attempt: usize) -> Result { // Simulate failure for first few attempts if attempt < 3 { Err(anyhow::anyhow!("Simulated failure #{}", attempt)) } else { Ok(format!("Success on attempt {}", attempt)) } } #[tokio::test] async fn test_financial_safety_mechanisms() -> Result<()> { let mut test_suite = SafetyTestSuite::new(); test_suite.setup().await?; // Test 1: Decimal precision safety let large_price = Price::from_f64(999999999.99)?; let small_quantity = Quantity::from_f64(0.000001)?; let product = large_price.to_f64() * small_quantity.to_f64(); assert!(product.is_finite(), "Large*small calculations should remain finite"); assert!(product > 0.0, "Product should be positive"); // Test 2: Overflow protection let max_safe_price = Price::from_f64(f64::MAX / 1000.0)?; let normal_quantity = Quantity::from_f64(500.0)?; let calculation = max_safe_price.to_f64() * normal_quantity.to_f64(); assert!(calculation.is_finite(), "Large calculations should not overflow"); // Test 3: Division by zero protection let zero_quantity = Quantity::ZERO; let price = Price::from_f64(100.0)?; // Should handle division by zero gracefully in calculations if zero_quantity.to_f64() != 0.0 { let _ratio = price.to_f64() / zero_quantity.to_f64(); } else { // Properly handled zero division tracing::info!("Zero division properly detected and avoided"); } // Test 4: NaN/Infinity handling let invalid_values = vec![f64::NAN, f64::INFINITY, f64::NEG_INFINITY]; for invalid_value in invalid_values { let price_result = Price::from_f64(invalid_value); match price_result { Ok(_) => tracing::warn!("Price type accepted invalid value: {}", invalid_value), Err(_) => { // Correctly rejected invalid value test_suite.increment_safety_checks(); } } } Ok(()) } #[tokio::test] async fn test_memory_safety_mechanisms() -> Result<()> { let mut test_suite = SafetyTestSuite::new(); test_suite.setup().await?; // Test 1: Large data structure handling let large_position_count = 10000; let mut positions = Vec::with_capacity(large_position_count); for i in 0..large_position_count { let position = TestPosition::new( &format!("SYMBOL_{}", i), 100.0 + i as f64, 50000.0, 50100.0, )?; positions.push(position); } assert_eq!(positions.len(), large_position_count, "Should handle large position collections"); // Test 2: Memory allocation limits let initial_positions = positions.len(); positions.reserve(1000); // Reserve additional space assert!(positions.capacity() >= initial_positions + 1000, "Memory reservation should work correctly"); // Test 3: Concurrent access safety let positions_arc = Arc::new(std::sync::Mutex::new(positions)); let mut handles = Vec::new(); for i in 0..5 { let positions_clone = Arc::clone(&positions_arc); let handle = tokio::spawn(async move { let mut positions = positions_clone.lock().unwrap(); positions.push(TestPosition::new( &format!("CONCURRENT_{}", i), 100.0, 50000.0, 50000.0, ).unwrap()); }); handles.push(handle); } // Wait for all concurrent operations for handle in handles { handle.await?; } let final_count = { let positions = positions_arc.lock().unwrap(); positions.len() }; assert_eq!(final_count, large_position_count + 5, "Concurrent operations should be thread-safe"); test_suite.increment_safety_checks(); Ok(()) } #[tokio::test] async fn test_concurrent_safety_mechanisms() -> Result<()> { let mut test_suite = SafetyTestSuite::new(); test_suite.setup().await?; // Test concurrent kill switch operations let kill_switch_triggered = Arc::new(AtomicBool::new(false)); let safety_check_counter = Arc::new(AtomicU64::new(0)); let mut handles = Vec::new(); // Spawn multiple tasks that might trigger safety mechanisms for i in 0..10 { let triggered_clone = Arc::clone(&kill_switch_triggered); let counter_clone = Arc::clone(&safety_check_counter); let handle = tokio::spawn(async move { // Simulate safety checks for j in 0..100 { counter_clone.fetch_add(1, Ordering::SeqCst); // Randomly trigger kill switch (simulate emergency) if i == 5 && j == 50 { triggered_clone.store(true, Ordering::SeqCst); } // Small delay to allow interleaving tokio::time::sleep(Duration::from_micros(10)).await; } }); handles.push(handle); } // Wait for all tasks to complete for handle in handles { handle.await?; } // Verify concurrent operations worked correctly let total_checks = safety_check_counter.load(Ordering::SeqCst); assert_eq!(total_checks, 1000, "All safety checks should have been recorded"); let was_triggered = kill_switch_triggered.load(Ordering::SeqCst); assert!(was_triggered, "Kill switch should have been triggered"); Ok(()) } #[tokio::test] async fn test_emergency_response_coordination() -> Result<()> { let mut test_suite = SafetyTestSuite::new(); test_suite.setup().await?; // Test emergency response test_suite.test_emergency_response().await?; // Verify emergency events were recorded let system_failure_count = test_suite.get_emergency_count(EmergencyType::SystemFailure); assert!(system_failure_count > 0, "Emergency response should have been triggered"); Ok(()) } #[tokio::test] async fn test_safety_mechanism_latency() -> Result<()> { let mut test_suite = SafetyTestSuite::new(); test_suite.setup().await?; // Test latency of various safety operations let operations = vec![ ("kill_switch_check", Box::new(|| async { // Simulate kill switch check tokio::time::sleep(Duration::from_nanos(100)).await; Ok(()) }) as Box std::pin::Pin> + Send>> + Send>), ("position_limit_check", Box::new(|| async { let position = TestPosition::new("LATENCY_TEST", 100.0, 50000.0, 50000.0)?; // Simulate position check tokio::time::sleep(Duration::from_nanos(200)).await; Ok(()) })), ("drawdown_check", Box::new(|| async { // Simulate drawdown calculation let _loss = Price::from_f64(-50.0)?; tokio::time::sleep(Duration::from_nanos(150)).await; Ok(()) })), ]; for (operation_name, operation) in operations { let start_time = Instant::now(); // Execute operation operation().await?; let latency = start_time.elapsed(); // Validate latency meets safety requirements assert_hft_latency(latency, test_suite.config.max_safety_latency_us); tracing::info!("Safety operation '{}' completed in {}μs", operation_name, latency.as_micros()); test_suite.increment_safety_checks(); } Ok(()) } #[tokio::test] async fn test_safety_under_load() -> Result<()> { let mut test_suite = SafetyTestSuite::new(); test_suite.setup().await?; // Test safety mechanisms under high load let load_operations = 1000; let start_time = Instant::now(); let mut tasks = Vec::new(); for i in 0..load_operations { let task = tokio::spawn(async move { // Simulate various safety checks let position = TestPosition::new( &format!("LOAD_TEST_{}", i), 100.0 + (i as f64 % 1000.0), 50000.0, 50000.0 + (i as f64 % 100.0), )?; // Simulate position safety check let market_value = position.market_value(); let _is_safe = market_value.to_f64() < 10000.0; Ok::<(), anyhow::Error>(()) }); tasks.push(task); } // Wait for all load operations to complete let results = futures::future::join_all(tasks).await; // Check for failures let mut success_count = 0; for result in results { match result { Ok(Ok(_)) => success_count += 1, Ok(Err(e)) => tracing::warn!("Load test operation failed: {}", e), Err(e) => tracing::error!("Load test task panicked: {}", e), } } let total_time = start_time.elapsed(); let operations_per_second = (load_operations as f64) / total_time.as_secs_f64(); tracing::info!("Load test completed: {}/{} operations successful, {} ops/sec", success_count, load_operations, operations_per_second); // Validate performance under load assert!(operations_per_second > 1000.0, "Safety mechanisms should handle >1000 ops/sec"); assert!(success_count >= load_operations * 95 / 100, "At least 95% of operations should succeed under load"); Ok(()) } #[tokio::test] async fn test_edge_case_safety_scenarios() -> Result<()> { let mut test_suite = SafetyTestSuite::new(); test_suite.setup().await?; // Test 1: Extreme position sizes let extreme_position = TestPosition::new( "EXTREME_TEST", f64::MAX / 1000000.0, // Very large but safe position 1.0, 1.01, )?; assert!(extreme_position.market_value().to_f64().is_finite(), "Extreme positions should have finite market value"); // Test 2: Zero and negative values let zero_position = TestPosition::new("ZERO_TEST", 0.0, 100.0, 100.0)?; assert_eq!(zero_position.quantity.to_f64(), 0.0, "Zero positions should be handled"); // Test 3: Very small decimal values let micro_position = TestPosition::new( "MICRO_TEST", 0.000001, 50000.123456789, 50000.123456790, )?; assert!(micro_position.unrealized_pnl.to_f64().is_finite(), "Micro positions should have finite PnL"); // Test 4: Rapid position updates let mut rapid_position = TestPosition::new("RAPID_TEST", 100.0, 50000.0, 50000.0)?; for i in 0..1000 { rapid_position.current_price = Price::from_f64(50000.0 + (i as f64 * 0.01))?; rapid_position.unrealized_pnl = Price::from_f64( (rapid_position.current_price.to_f64() - rapid_position.avg_price.to_f64()) * rapid_position.quantity.to_f64() )?; // Verify each update is valid assert!(rapid_position.unrealized_pnl.to_f64().is_finite(), "Rapid updates should maintain finite values"); } test_suite.increment_safety_checks(); Ok(()) } #[tokio::test] async fn test_safety_configuration_validation() -> Result<()> { let mut test_suite = SafetyTestSuite::new(); // Test invalid safety configurations let invalid_configs = vec![ ("negative_position_limit", SafetyTestConfig { test_position_limits: TestPositionLimits { max_position_per_symbol: -1000.0, // Invalid negative limit ..Default::default() }, ..Default::default() }), ("zero_drawdown_limit", SafetyTestConfig { test_drawdown_limits: TestDrawdownLimits { max_daily_loss: 0.0, // Invalid zero limit ..Default::default() }, ..Default::default() }), ("extreme_latency_requirement", SafetyTestConfig { max_safety_latency_us: 0, // Impossible latency requirement ..Default::default() }), ]; for (config_name, invalid_config) in invalid_configs { test_suite.config = invalid_config; // Should handle invalid configurations gracefully let setup_result = test_suite.setup().await; match setup_result { Ok(_) => { tracing::warn!("Invalid config '{}' was accepted", config_name); } Err(_) => { tracing::info!("Invalid config '{}' was correctly rejected", config_name); } } } Ok(()) } #[tokio::test] async fn test_comprehensive_safety_integration() -> Result<()> { let mut test_suite = SafetyTestSuite::new(); test_suite.setup().await?; // Execute comprehensive safety test scenario let scenario_start = Instant::now(); // 1. Test normal operations let safe_position = test_suite.create_safe_position()?; assert!(safe_position.market_value().to_f64() > 0.0, "Safe position should have positive value"); // 2. Test limit violations let violating_position = test_suite.create_violating_position()?; test_suite.test_position_limits().await?; // 3. Test drawdown scenarios test_suite.test_drawdown_protection().await?; // 4. Test emergency responses test_suite.test_emergency_response().await?; // 5. Test kill switch functionality test_suite.test_kill_switch_activation().await?; let scenario_time = scenario_start.elapsed(); // Validate overall scenario performance assert!(scenario_time.as_secs() < 10, "Comprehensive safety test should complete within 10 seconds"); // Verify all safety mechanisms were tested let total_checks = test_suite.total_safety_checks.load(Ordering::SeqCst); assert!(total_checks > 0, "Safety checks should have been performed"); // Verify emergency events were recorded let total_events = if let Ok(events) = test_suite.emergency_events.lock() { events.len() } else { 0 }; tracing::info!("Comprehensive safety test completed: {} safety checks, {} emergency events in {}ms", total_checks, total_events, scenario_time.as_millis()); Ok(()) } // ========== UTILITY FUNCTIONS FOR SAFETY TESTS ========== /// Create test emergency scenario fn create_test_emergency_scenario(severity: RiskSeverity) -> Result { let event_type = match severity { RiskSeverity::Critical => EmergencyType::SystemFailure, RiskSeverity::High => EmergencyType::PositionLimit, RiskSeverity::Medium => EmergencyType::RiskViolation, RiskSeverity::Low => EmergencyType::NetworkFailure, }; EmergencyEvent::new(event_type, severity, &format!("Test emergency: {:?}", severity)) } /// Validate safety mechanism response time fn validate_safety_response_time(start_time: Instant, max_latency_us: u64, operation: &str) -> Result<()> { let latency = start_time.elapsed(); assert_hft_latency(latency, max_latency_us); tracing::debug!("Safety operation '{}' completed in {}μs", operation, latency.as_micros()); Ok(()) } /// Create stress test data set for safety validation fn create_safety_stress_dataset(size: usize) -> Result> { let mut positions = Vec::with_capacity(size); for i in 0..size { let symbol = format!("STRESS_{}", i); let quantity = 100.0 + (i as f64 % 1000.0); let base_price = 50000.0; let current_price = base_price + ((i as f64 % 100.0) - 50.0); // ±50 price variation positions.push(TestPosition::new(&symbol, quantity, base_price, current_price)?); } Ok(positions) }