foxhunt/tests/failure_scenario_tests.rs

//! Failure Scenario Integration Tests for Foxhunt HFT System
//!
//! This module tests the system's resilience and recovery capabilities under various failure conditions:
//!
//! ## Failure Scenarios Covered:
//! 1. **Kill Switch Activation**: Emergency shutdown procedures
//! 2. **Trading Gate Guards**: Order processing gates under kill switch
//! 3. **Scoped Kill Switches**: Symbol/Account/Strategy level blocks
//!
//! ## Recovery Validation:
//! - System gracefully handles all failure modes
//! - Recovery procedures restore full functionality
//! - Kill switch activation is immediate
//! - Trading gates prevent order submission when active

#![warn(missing_docs)]
#![warn(clippy::all)]
#![allow(clippy::too_many_arguments)]
#![allow(clippy::type_complexity)]
#![allow(unused_crate_dependencies)]

use std::sync::Arc;
use std::time::{Duration, Instant};
use tracing::{info, warn};

// Core system imports - use full paths since types aren't re-exported from crate root
use risk::AtomicKillSwitch;
use risk::risk_types::KillSwitchScope;
use risk::safety::KillSwitchConfig;
use risk::safety::trading_gate::TradingGate;

/// Failure scenario test configuration
#[derive(Debug, Clone)]
pub struct FailureTestConfig {
    /// Redis connection string for kill switch
    pub redis_url: String,
    /// Test timeout duration
    pub failure_timeout: Duration,
    /// Maximum acceptable activation time
    pub max_activation_time: Duration,
}

impl Default for FailureTestConfig {
    fn default() -> Self {
        Self {
            redis_url: std::env::var("TEST_REDIS_URL")
                .unwrap_or_else(|_| "redis://localhost:6379/2".to_string()),
            failure_timeout: Duration::from_secs(30),
            max_activation_time: Duration::from_millis(100),
        }
    }
}

/// Failure test harness for managing kill switch scenarios
pub struct FailureTestHarness {
    config: FailureTestConfig,
    kill_switch: Arc<AtomicKillSwitch>,
    trading_gate: Arc<TradingGate>,
}

impl FailureTestHarness {
    /// Create a new failure test harness
    pub async fn new() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
        let config = FailureTestConfig::default();

        info!("Initializing failure scenario test harness");

        // Initialize kill switch with Redis backend
        let kill_switch_config = KillSwitchConfig::default();
        let kill_switch = Arc::new(
            AtomicKillSwitch::new(kill_switch_config, config.redis_url.clone()).await?
        );

        // Initialize trading gate
        let trading_gate = Arc::new(TradingGate::new(kill_switch.clone()));

        info!("Failure scenario test harness initialized successfully");

        Ok(Self {
            config,
            kill_switch,
            trading_gate,
        })
    }

    /// Test kill switch activation and emergency procedures
    pub async fn test_kill_switch_activation(
        &self,
    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        info!("STARTING: Kill Switch Activation Test");

        let start_time = Instant::now();

        // Step 1: Verify normal operations (kill switch inactive)
        info!("Step 1: Verifying normal trading operations...");
        let is_active = self.kill_switch.is_active().await?;
        if is_active {
            return Err("Kill switch should be inactive at start".into());
        }

        // Verify trading gate allows orders
        let gate_result = self.trading_gate.pre_order_gate("AAPL", None);
        if gate_result.is_err() {
            return Err("Trading gate should allow orders when kill switch inactive".into());
        }

        // Step 2: Activate kill switch
        info!("Step 2: Activating emergency kill switch...");
        let activation_start = Instant::now();
        self.kill_switch
            .activate(
                KillSwitchScope::Global,
                "Integration test emergency scenario".to_string(),
                "test_user".to_string(),
                false,
            )
            .await?;
        let activation_time = activation_start.elapsed();

        // Step 3: Verify kill switch is active
        info!("Step 3: Verifying kill switch activation...");
        let is_active = self.kill_switch.is_active().await?;
        if !is_active {
            return Err("Kill switch should be active after activation".into());
        }

        // Step 4: Verify all trading halts immediately
        info!("Step 4: Verifying immediate trading halt...");
        let gate_result = self.trading_gate.pre_order_gate("AAPL", None);
        if gate_result.is_ok() {
            return Err("Trading gate should block orders when kill switch active".into());
        }

        // Verify emergency check
        if self.trading_gate.emergency_check() {
            return Err("Emergency check should return false when kill switch active".into());
        }

        // Step 5: Test recovery from kill switch
        info!("Step 5: Testing recovery from kill switch...");
        self.kill_switch.deactivate(
            KillSwitchScope::Global,
            "Test completed".to_string(),
        ).await?;

        let is_active = self.kill_switch.is_active().await?;
        if is_active {
            return Err("Kill switch should be inactive after deactivation".into());
        }

        // Verify trading gate allows orders again
        let gate_result = self.trading_gate.pre_order_gate("AAPL", None);
        if gate_result.is_err() {
            return Err("Trading gate should allow orders after kill switch deactivation".into());
        }

        let test_duration = start_time.elapsed();

        info!("KILL SWITCH ACTIVATION TEST COMPLETED");
        info!("  Activation Time: {:?} (target: <{:?})", activation_time, self.config.max_activation_time);
        info!("  Total Duration: {:?}", test_duration);

        if activation_time > self.config.max_activation_time {
            warn!("Kill switch activation took longer than target: {:?} > {:?}",
                activation_time, self.config.max_activation_time);
        }

        Ok(())
    }

    /// Test scoped kill switch activation (symbol-level)
    pub async fn test_scoped_kill_switch(
        &self,
    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        info!("STARTING: Scoped Kill Switch Test");

        // Step 1: Activate kill switch for specific symbol
        info!("Step 1: Activating kill switch for AAPL...");
        self.kill_switch
            .activate(
                KillSwitchScope::Symbol("AAPL".to_string()),
                "Test symbol-level block".to_string(),
                "test_user".to_string(),
                false,
            )
            .await?;

        // Step 2: Verify AAPL is blocked
        info!("Step 2: Verifying AAPL is blocked...");
        let aapl_result = self.trading_gate.pre_order_gate("AAPL", None);
        if aapl_result.is_ok() {
            return Err("AAPL should be blocked by scoped kill switch".into());
        }

        // Step 3: Verify MSFT is still allowed
        info!("Step 3: Verifying MSFT is still allowed...");
        let msft_result = self.trading_gate.pre_order_gate("MSFT", None);
        if msft_result.is_err() {
            return Err("MSFT should not be affected by AAPL kill switch".into());
        }

        // Step 4: Deactivate scoped kill switch
        info!("Step 4: Deactivating AAPL kill switch...");
        self.kill_switch.deactivate(
            KillSwitchScope::Symbol("AAPL".to_string()),
            "Test completed".to_string(),
        ).await?;

        // Step 5: Verify AAPL is allowed again
        info!("Step 5: Verifying AAPL is allowed again...");
        let aapl_result = self.trading_gate.pre_order_gate("AAPL", None);
        if aapl_result.is_err() {
            return Err("AAPL should be allowed after deactivation".into());
        }

        info!("SCOPED KILL SWITCH TEST COMPLETED");

        Ok(())
    }

    /// Test trading gate performance under normal conditions
    pub async fn test_trading_gate_performance(
        &self,
    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        info!("STARTING: Trading Gate Performance Test");

        let num_checks = 10_000;
        let start_time = Instant::now();

        for _ in 0..num_checks {
            let _ = self.trading_gate.pre_order_gate("AAPL", None);
        }

        let total_duration = start_time.elapsed();
        let avg_latency_ns = total_duration.as_nanos() / num_checks;

        info!("TRADING GATE PERFORMANCE TEST COMPLETED");
        info!("  Total Checks: {}", num_checks);
        info!("  Total Duration: {:?}", total_duration);
        info!("  Average Latency: {}ns", avg_latency_ns);

        // HFT compliance: each check should be sub-microsecond
        if avg_latency_ns > 1000 {
            warn!("Trading gate latency exceeds 1 microsecond target: {}ns", avg_latency_ns);
        }

        Ok(())
    }

    /// Test multiple concurrent gate checks
    pub async fn test_concurrent_gate_checks(
        &self,
    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        info!("STARTING: Concurrent Gate Checks Test");

        let num_tasks = 100;
        let checks_per_task = 1000;

        let mut handles = Vec::new();

        for task_id in 0..num_tasks {
            let gate = self.trading_gate.clone();
            let handle = tokio::spawn(async move {
                for _ in 0..checks_per_task {
                    let _ = gate.pre_order_gate("AAPL", None);
                }
                task_id
            });
            handles.push(handle);
        }

        // Wait for all tasks to complete
        for handle in handles {
            handle.await?;
        }

        info!("CONCURRENT GATE CHECKS TEST COMPLETED");
        info!("  Tasks: {}", num_tasks);
        info!("  Checks per Task: {}", checks_per_task);
        info!("  Total Checks: {}", num_tasks * checks_per_task);

        Ok(())
    }

    /// Test batch symbol gate checking
    pub async fn test_batch_symbol_gate(
        &self,
    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        info!("STARTING: Batch Symbol Gate Test");

        let symbols = vec![
            "AAPL".to_string(),
            "GOOGL".to_string(),
            "MSFT".to_string(),
            "TSLA".to_string(),
        ];

        // Step 1: Test with no kill switches
        info!("Step 1: Testing batch gate with no kill switches...");
        let allowed = self.trading_gate.batch_symbol_gate(&symbols)?;
        if allowed.len() != symbols.len() {
            return Err(format!("Expected {} allowed symbols, got {}", symbols.len(), allowed.len()).into());
        }

        // Step 2: Activate kill switch for AAPL
        info!("Step 2: Activating kill switch for AAPL...");
        self.kill_switch
            .activate(
                KillSwitchScope::Symbol("AAPL".to_string()),
                "Test batch filtering".to_string(),
                "test_user".to_string(),
                false,
            )
            .await?;

        // Step 3: Test batch gate with one symbol blocked
        info!("Step 3: Testing batch gate with AAPL blocked...");
        let allowed = self.trading_gate.batch_symbol_gate(&symbols)?;
        if allowed.len() != symbols.len() - 1 {
            return Err(format!("Expected {} allowed symbols, got {}", symbols.len() - 1, allowed.len()).into());
        }
        if allowed.contains(&"AAPL".to_string()) {
            return Err("AAPL should not be in allowed symbols".into());
        }

        // Step 4: Cleanup
        info!("Step 4: Cleaning up...");
        self.kill_switch.deactivate(
            KillSwitchScope::Symbol("AAPL".to_string()),
            "Test completed".to_string(),
        ).await?;

        info!("BATCH SYMBOL GATE TEST COMPLETED");

        Ok(())
    }
}

// Integration test runner
#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_kill_switch_activation() {
        let harness = FailureTestHarness::new()
            .await
            .expect("Failed to initialize failure test harness");

        harness
            .test_kill_switch_activation()
            .await
            .expect("Kill switch activation test failed");
    }

    #[tokio::test]
    async fn test_scoped_kill_switch() {
        let harness = FailureTestHarness::new()
            .await
            .expect("Failed to initialize failure test harness");

        harness
            .test_scoped_kill_switch()
            .await
            .expect("Scoped kill switch test failed");
    }

    #[tokio::test]
    async fn test_trading_gate_performance() {
        let harness = FailureTestHarness::new()
            .await
            .expect("Failed to initialize failure test harness");

        harness
            .test_trading_gate_performance()
            .await
            .expect("Trading gate performance test failed");
    }

    #[tokio::test]
    async fn test_concurrent_gate_checks() {
        let harness = FailureTestHarness::new()
            .await
            .expect("Failed to initialize failure test harness");

        harness
            .test_concurrent_gate_checks()
            .await
            .expect("Concurrent gate checks test failed");
    }

    #[tokio::test]
    async fn test_batch_symbol_gate() {
        let harness = FailureTestHarness::new()
            .await
            .expect("Failed to initialize failure test harness");

        harness
            .test_batch_symbol_gate()
            .await
            .expect("Batch symbol gate test failed");
    }
}