//! Kill Switch Integration for Trading Service //! //! Integrates the atomic kill switch system with the trading service //! to provide regulatory-compliant sub-100ms emergency shutdown capability. use std::sync::Arc; use std::time::Duration; use anyhow::{Context, Result}; use tokio::sync::RwLock; use tracing::{error, info, warn}; use risk::safety::emergency_response::EmergencyResponseSystem; use risk::safety::kill_switch::AtomicKillSwitch; use risk::safety::trading_gate::TradingGate; use risk::safety::unix_socket_kill_switch::UnixSocketKillSwitch; use risk::safety::{EmergencyResponseConfig, KillSwitchConfig}; use crate::error::{TradingServiceError, TradingServiceResult}; /// Kill switch integration for the trading service pub struct TradingServiceKillSwitch { /// The atomic kill switch instance pub kill_switch: Arc, /// Trading gate for order validation pub trading_gate: Arc, /// Unix socket interface for external control pub unix_socket_controller: Arc>>, /// Emergency response system pub emergency_response: Arc, } impl std::fmt::Debug for TradingServiceKillSwitch { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("TradingServiceKillSwitch") .field("kill_switch", &self.kill_switch) .field("trading_gate", &self.trading_gate) .field( "unix_socket_controller", &">>>", ) .field("emergency_response", &">") .finish() } } impl TradingServiceKillSwitch { /// Initialize the kill switch system for the trading service pub async fn new(redis_url: String) -> Result { info!("Initializing trading service kill switch system"); // Create kill switch configuration let kill_switch_config = KillSwitchConfig { enabled: true, global_channel: "foxhunt:safety:kill_switch:global".to_string(), strategy_channel_prefix: "foxhunt:safety:kill_switch:strategy".to_string(), symbol_channel_prefix: "foxhunt:safety:kill_switch:symbol".to_string(), auto_recovery_enabled: false, // Disable auto-recovery for trading service auto_recovery_delay: Duration::from_secs(300), }; // Initialize atomic kill switch let kill_switch = Arc::new( AtomicKillSwitch::new(kill_switch_config, redis_url.clone()) .await .context("Failed to create atomic kill switch")?, ); // Create trading gate let trading_gate = Arc::new(TradingGate::new(Arc::clone(&kill_switch))); // Initialize emergency response system let emergency_config = EmergencyResponseConfig::default(); let emergency_response = Arc::new( EmergencyResponseSystem::new(emergency_config, redis_url, Arc::clone(&kill_switch)) .await .context("Failed to create emergency response system")?, ); // Unix socket controller will be initialized when started let unix_socket_controller = Arc::new(RwLock::new(None)); Ok(Self { kill_switch, trading_gate, unix_socket_controller, emergency_response, }) } /// Start the kill switch monitoring and external interfaces pub async fn start_monitoring(&self) -> Result<()> { info!("Starting kill switch monitoring systems"); // Start atomic kill switch monitoring self.kill_switch .start_monitoring() .await .context("Failed to start kill switch monitoring")?; // Start emergency response monitoring self.emergency_response .start_monitoring() .await .context("Failed to start emergency response monitoring")?; // Initialize Unix socket interface let socket_path = std::env::var("KILL_SWITCH_SOCKET_PATH") .unwrap_or_else(|_| "/tmp/foxhunt/kill_switch.sock".to_string()); let mut unix_socket = UnixSocketKillSwitch::new(socket_path, Arc::clone(&self.kill_switch)) .await .context("Failed to create Unix socket kill switch")?; // Setup emergency shutdown signal handlers unix_socket .setup_emergency_shutdown_signals() .await .context("Failed to setup emergency shutdown signals")?; // Start Unix socket listener unix_socket .start_listener() .await .context("Failed to start Unix socket listener")?; // Store the Unix socket controller { let mut controller = self.unix_socket_controller.write().await; *controller = Some(unix_socket); } info!("Kill switch monitoring systems started successfully"); Ok(()) } /// Stop the kill switch monitoring systems pub async fn stop_monitoring(&self) -> Result<()> { info!("Stopping kill switch monitoring systems"); // Stop Unix socket listener { let mut controller = self.unix_socket_controller.write().await; if let Some(ref mut unix_socket) = *controller { if let Err(e) = unix_socket.stop_listener().await { warn!("Error stopping Unix socket listener: {}", e); } } *controller = None; } // Stop monitoring systems if let Err(e) = self.emergency_response.stop_monitoring().await { warn!("Error stopping emergency response monitoring: {}", e); } if let Err(e) = self.kill_switch.stop_monitoring().await { warn!("Error stopping kill switch monitoring: {}", e); } info!("Kill switch monitoring systems stopped"); Ok(()) } /// Check if trading is allowed for the given symbol and account #[inline(always)] pub fn check_trading_allowed( &self, symbol: &str, account: Option<&str>, ) -> TradingServiceResult<()> { self.trading_gate .pre_order_gate(symbol, account) .map_err(|e| TradingServiceError::RiskViolation { violation_type: "kill_switch".to_string(), message: e.to_string(), }) } /// Emergency shutdown - activate global kill switch immediately pub async fn emergency_shutdown(&self, reason: String) -> Result<()> { error!("🚨 EMERGENCY SHUTDOWN TRIGGERED: {}", reason); // Activate global kill switch self.kill_switch .activate_global(reason.clone(), "trading-service".to_string()) .await .context("Failed to activate global kill switch")?; // Handle manual emergency in emergency response system self.emergency_response .handle_manual_emergency("trading-service".to_string(), reason) .await .context("Failed to handle manual emergency")?; error!("🚨 EMERGENCY SHUTDOWN COMPLETE"); Ok(()) } /// Get kill switch status and metrics pub async fn get_status(&self) -> Result { let is_active = self .kill_switch .is_active() .await .context("Failed to get kill switch status")?; let is_healthy = self .kill_switch .is_healthy() .await .context("Failed to get kill switch health")?; let (checks, commands) = self.kill_switch.get_metrics(); let (error_rate, consecutive_failures) = self.kill_switch.get_health_metrics(); let is_emergency_active = { let controller = self.unix_socket_controller.read().await; controller .as_ref() .map(|c| c.is_emergency_shutdown_active()) .unwrap_or(false) }; Ok(KillSwitchStatus { is_active, is_healthy, is_emergency_active, total_checks: checks, total_commands: commands, error_rate, consecutive_failures, }) } /// Get the trading gate for use in order processing pub fn trading_gate(&self) -> &Arc { &self.trading_gate } /// Get the underlying kill switch for advanced operations pub fn kill_switch(&self) -> &Arc { &self.kill_switch } /// Check if emergency shutdown is active pub async fn is_emergency_shutdown_active(&self) -> bool { let controller = self.unix_socket_controller.read().await; controller .as_ref() .map(|c| c.is_emergency_shutdown_active()) .unwrap_or(false) } } /// Kill switch status information #[derive(Debug, Clone)] pub struct KillSwitchStatus { pub is_active: bool, pub is_healthy: bool, pub is_emergency_active: bool, pub total_checks: u64, pub total_commands: u64, pub error_rate: f64, pub consecutive_failures: u64, } /// Utility functions for integrating kill switch with trading operations impl TradingServiceKillSwitch { /// Validate order with comprehensive kill switch checks pub async fn validate_order_with_kill_switch( &self, symbol: &str, account: &str, strategy_id: Option<&str>, ) -> TradingServiceResult<()> { // Use comprehensive gate check self.trading_gate .comprehensive_order_gate(symbol, account, strategy_id) .map_err(|e| TradingServiceError::RiskViolation { violation_type: "kill_switch_comprehensive".to_string(), message: e.to_string(), }) } /// Check if market data processing should continue pub fn validate_market_data_processing(&self, symbol: &str) -> TradingServiceResult<()> { self.trading_gate .market_data_gate(symbol) .map_err(|e| TradingServiceError::RiskViolation { violation_type: "kill_switch_market_data".to_string(), message: e.to_string(), }) } /// Final execution gate - last check before sending order to broker pub fn execution_gate_check(&self, symbol: &str, account: &str) -> TradingServiceResult<()> { self.trading_gate .execution_gate(symbol, account) .map_err(|e| TradingServiceError::RiskViolation { violation_type: "kill_switch_execution".to_string(), message: e.to_string(), }) } /// Batch check for multiple symbols (useful for portfolio operations) pub fn batch_symbol_check(&self, symbols: &[String]) -> TradingServiceResult> { self.trading_gate.batch_symbol_gate(symbols).map_err(|e| { TradingServiceError::RiskViolation { violation_type: "kill_switch_batch".to_string(), message: e.to_string(), } }) } } /// Macro for easy integration of kill switch checks in trading service methods #[macro_export] macro_rules! kill_switch_check { ($kill_switch:expr, $symbol:expr) => { if let Err(e) = $kill_switch.check_trading_allowed($symbol, None) { return Err(e); } }; ($kill_switch:expr, $symbol:expr, $account:expr) => { if let Err(e) = $kill_switch.check_trading_allowed($symbol, Some($account)) { return Err(e); } }; } #[cfg(test)] mod tests { use super::*; use crate::test_utils::TestConfig; /// Helper to check if Redis is available (connects to Docker Redis) async fn is_redis_available() -> bool { match redis::Client::open("redis://127.0.0.1:6379") { Ok(client) => client.get_multiplexed_tokio_connection().await.is_ok(), Err(_) => false, } } #[tokio::test] async fn test_kill_switch_integration_creation() { if !is_redis_available().await { tracing::info!("Skipping test: Redis not available"); return; } let result = TradingServiceKillSwitch::new("redis://localhost:6379".to_string()).await; assert!(result.is_ok()); let kill_switch_integration = result.expect("Kill switch integration should be created successfully"); let status = kill_switch_integration .get_status() .await .expect("Should get status successfully"); // Initially should not be active assert!(!status.is_active); assert!(status.is_healthy); assert!(!status.is_emergency_active); } #[tokio::test] async fn test_trading_validation() { if !is_redis_available().await { tracing::info!("Skipping test: Redis not available"); return; } let test_config = TestConfig::new(); let kill_switch_integration = TradingServiceKillSwitch::new("redis://localhost:6379".to_string()) .await .expect("Kill switch integration should be created for trading validation test"); // Should allow trading initially let result = kill_switch_integration.check_trading_allowed( test_config.primary_symbol(), Some(&test_config.default_account), ); assert!(result.is_ok()); // Should allow comprehensive validation let result = kill_switch_integration .validate_order_with_kill_switch( test_config.primary_symbol(), &test_config.default_account, Some(&test_config.default_strategy), ) .await; assert!(result.is_ok()); } #[tokio::test] async fn test_emergency_shutdown() { if !is_redis_available().await { tracing::info!("Skipping test: Redis not available"); return; } let kill_switch_integration = TradingServiceKillSwitch::new("redis://localhost:6379".to_string()) .await .expect("Kill switch integration should be created for emergency shutdown test"); // Trigger emergency shutdown let result = kill_switch_integration .emergency_shutdown("Test emergency".to_string()) .await; assert!(result.is_ok()); // Should now block trading let status = kill_switch_integration .get_status() .await .expect("Should get status after emergency shutdown"); assert!(status.is_active); } #[tokio::test] async fn test_batch_symbol_check() { if !is_redis_available().await { tracing::info!("Skipping test: Redis not available"); return; } let test_config = TestConfig::new(); let kill_switch_integration = TradingServiceKillSwitch::new("redis://localhost:6379".to_string()) .await .expect("Kill switch integration should be created for batch symbol check test"); let symbols = test_config.symbol_subset(3); let result = kill_switch_integration.batch_symbol_check(&symbols); assert!(result.is_ok()); let allowed_symbols = result.expect("Batch symbol check should return allowed symbols"); assert_eq!(allowed_symbols.len(), 3); } #[tokio::test] async fn test_monitoring_lifecycle() { if !is_redis_available().await { tracing::info!("Skipping test: Redis not available"); return; } let kill_switch_integration = TradingServiceKillSwitch::new("redis://localhost:6379".to_string()) .await .expect("Kill switch integration should be created for monitoring lifecycle test"); // Start monitoring (may fail in test environment due to /var/run/kill_switch permissions) let start_result = kill_switch_integration.start_monitoring().await; // If we can start monitoring, test stopping if start_result.is_ok() { let stop_result = kill_switch_integration.stop_monitoring().await; assert!(stop_result.is_ok()); } else { // Expected in test environment - /var/run may not be writable println!( "Monitoring start failed (expected in test environment): {:?}", start_result ); } } }