foxhunt/tests/integration/broker_failover.rs

//! Multi-Broker Failover and Smart Routing Validation Tests
//!
//! These tests validate the broker failover and smart routing capabilities by testing:
//! - Automatic failover between Interactive Brokers and ICMarkets
//! - Smart order routing based on latency and availability
//! - Connection recovery and order re-routing scenarios
//! - Load balancing across multiple broker connections
//! - Graceful degradation when brokers become unavailable
//!
//! NOTE: These tests simulate real broker failover scenarios and validate
//! that the system maintains trading capability even when individual brokers fail.
#![allow(unused_crate_dependencies)]

use std::env;
use std::time::Duration;
use std::collections::HashMap;
use std::sync::Arc;
use tokio::time::timeout;
use tokio::sync::{RwLock, Mutex};
use tracing::{info, warn, error};

use trading_engine::brokers::interactive_brokers::InteractiveBrokersClient;
use trading_engine::brokers::icmarkets::ICMarketsClient;
use trading_engine::brokers::config::{InteractiveBrokersConfig, ICMarketsConfig};
use trading_engine::brokers::routing::router::SmartOrderRouter;
use trading_engine::brokers::routing::decision::RoutingDecision;
use trading_engine::brokers::routing::metrics::LatencyMetrics;
use trading_engine::trading::data_interface::{BrokerInterface, BrokerConnectionStatus};
use trading_engine::prelude::{TradingOrder, OrderSide};
use trading_engine::trading_operations::OrderType;
use common::TimeInForce;

/// Mock broker for testing failover scenarios
#[derive(Debug, Clone)]
pub struct MockBroker {
    name: String,
    is_available: Arc<RwLock<bool>>,
    latency_ms: Arc<RwLock<u64>>,
    order_count: Arc<RwLock<u64>>,
    failure_rate: Arc<RwLock<f64>>, // 0.0 = never fail, 1.0 = always fail
}

impl MockBroker {
    pub fn new(name: &str, initial_latency_ms: u64) -> Self {
        Self {
            name: name.to_string(),
            is_available: Arc::new(RwLock::new(true)),
            latency_ms: Arc::new(RwLock::new(initial_latency_ms)),
            order_count: Arc::new(RwLock::new(0)),
            failure_rate: Arc::new(RwLock::new(0.0)),
        }
    }
    
    pub async fn set_availability(&self, available: bool) {
        *self.is_available.write().await = available;
    }
    
    pub async fn set_latency(&self, latency_ms: u64) {
        *self.latency_ms.write().await = latency_ms;
    }
    
    pub async fn set_failure_rate(&self, rate: f64) {
        *self.failure_rate.write().await = rate.clamp(0.0, 1.0);
    }
    
    pub async fn get_order_count(&self) -> u64 {
        *self.order_count.read().await
    }
    
    pub async fn simulate_order_execution(&self, order: &TradingOrder) -> Result<String, String> {
        // Check availability
        if !*self.is_available.read().await {
            return Err(format!("Broker {} is not available", self.name));
        }
        
        // Simulate latency
        let latency = *self.latency_ms.read().await;
        tokio::time::sleep(Duration::from_millis(latency)).await;
        
        // Check failure rate
        let failure_rate = *self.failure_rate.read().await;
        if rand::random::<f64>() < failure_rate {
            return Err(format!("Broker {} execution failed (simulated)", self.name));
        }
        
        // Increment order count
        *self.order_count.write().await += 1;
        
        let execution_id = format!("{}_{}", self.name, uuid::Uuid::new_v4());
        Ok(execution_id)
    }
}

/// Multi-broker manager for testing failover scenarios
#[derive(Debug)]
pub struct MultiBrokerManager {
    brokers: Vec<MockBroker>,
    routing_metrics: Arc<RwLock<HashMap<String, LatencyMetrics>>>,
    primary_broker: Arc<RwLock<Option<String>>>,
    failover_threshold_ms: u64,
    health_check_interval: Duration,
}

impl MultiBrokerManager {
    pub fn new(failover_threshold_ms: u64) -> Self {
        Self {
            brokers: Vec::new(),
            routing_metrics: Arc::new(RwLock::new(HashMap::new())),
            primary_broker: Arc::new(RwLock::new(None)),
            failover_threshold_ms,
            health_check_interval: Duration::from_secs(5),
        }
    }
    
    pub fn add_broker(&mut self, broker: MockBroker) {
        // Set first broker as primary
        if self.brokers.is_empty() {
            tokio::spawn({
                let primary = self.primary_broker.clone();
                let name = broker.name.clone();
                async move {
                    *primary.write().await = Some(name);
                }
            });
        }
        
        self.brokers.push(broker);
    }
    
    pub async fn execute_order_with_failover(&self, order: &TradingOrder) -> Result<(String, String), String> {
        // Try primary broker first
        if let Some(primary_name) = self.primary_broker.read().await.clone() {
            if let Some(primary_broker) = self.brokers.iter().find(|b| b.name == primary_name) {
                match primary_broker.simulate_order_execution(order).await {
                    Ok(execution_id) => {
                        info!("✅ Order executed on primary broker {}: {}", primary_name, execution_id);
                        return Ok((primary_name, execution_id));
                    }
                    Err(e) => {
                        warn!("⚠️  Primary broker {} failed: {}", primary_name, e);
                    }
                }
            }
        }
        
        // Try failover brokers
        for broker in &self.brokers {
            let is_primary = Some(broker.name.clone()) == *self.primary_broker.read().await;
            if is_primary {
                continue; // Already tried primary
            }
            
            match broker.simulate_order_execution(order).await {
                Ok(execution_id) => {
                    warn!("🔄 Order executed on failover broker {}: {}", broker.name, execution_id);
                    
                    // Update primary broker to successful failover broker
                    *self.primary_broker.write().await = Some(broker.name.clone());
                    
                    return Ok((broker.name.clone(), execution_id));
                }
                Err(e) => {
                    warn!("⚠️  Failover broker {} also failed: {}", broker.name, e);
                }
            }
        }
        
        Err("All brokers failed - no execution possible".to_string())
    }
    
    pub async fn get_broker_health_status(&self) -> HashMap<String, bool> {
        let mut status = HashMap::new();
        
        for broker in &self.brokers {
            let is_available = *broker.is_available.read().await;
            status.insert(broker.name.clone(), is_available);
        }
        
        status
    }
    
    pub async fn get_routing_statistics(&self) -> HashMap<String, u64> {
        let mut stats = HashMap::new();
        
        for broker in &self.brokers {
            let count = broker.get_order_count().await;
            stats.insert(broker.name.clone(), count);
        }
        
        stats
    }
    
    pub async fn simulate_broker_failure(&self, broker_name: &str) {
        if let Some(broker) = self.brokers.iter().find(|b| b.name == broker_name) {
            broker.set_availability(false).await;
            warn!("🔥 Simulated failure for broker: {}", broker_name);
        }
    }
    
    pub async fn simulate_broker_recovery(&self, broker_name: &str) {
        if let Some(broker) = self.brokers.iter().find(|b| b.name == broker_name) {
            broker.set_availability(true).await;
            info!("🔄 Simulated recovery for broker: {}", broker_name);
        }
    }
}

/// Helper function to create test trading order
fn create_test_order(symbol: &str, side: OrderSide, quantity: i64, price: f64) -> TradingOrder {
    TradingOrder {
        id: OrderId::new(),
        symbol: Symbol::new(symbol.to_string()),
        side,
        quantity: Quantity::from_f64(quantity as f64).unwrap_or_default(),
        price: Price::from_f64(price).unwrap_or_default(),
        order_type: OrderType::Limit,
        time_in_force: TimeInForce::Day,
        timestamp: chrono::Utc::now(),
        metadata: HashMap::new(),
    }
}

#[tokio::test]
async fn test_basic_broker_failover() {
    info!("🔄 Testing basic broker failover scenario");
    
    let mut manager = MultiBrokerManager::new(1000); // 1 second failover threshold
    
    // Add test brokers
    manager.add_broker(MockBroker::new("primary_broker", 50));   // Fast primary
    manager.add_broker(MockBroker::new("backup_broker", 100));   // Slower backup
    manager.add_broker(MockBroker::new("tertiary_broker", 200)); // Slowest tertiary
    
    let test_order = create_test_order("AAPL", OrderSide::Buy, 100, 150.50);
    
    // Normal execution (should use primary)
    let result1 = manager.execute_order_with_failover(&test_order).await;
    match result1 {
        Ok((broker_name, execution_id)) => {
            assert_eq!(broker_name, "primary_broker");
            info!("✅ Normal execution used primary broker: {}", execution_id);
        }
        Err(e) => {
            panic!("❌ Normal execution should succeed: {}", e);
        }
    }
    
    // Simulate primary broker failure
    manager.simulate_broker_failure("primary_broker").await;
    
    let test_order2 = create_test_order("MSFT", OrderSide::Sell, 50, 300.25);
    
    // Should failover to backup
    let result2 = manager.execute_order_with_failover(&test_order2).await;
    match result2 {
        Ok((broker_name, execution_id)) => {
            assert_eq!(broker_name, "backup_broker");
            info!("✅ Failover execution used backup broker: {}", execution_id);
        }
        Err(e) => {
            panic!("❌ Failover execution should succeed: {}", e);
        }
    }
    
    // Simulate backup broker failure too
    manager.simulate_broker_failure("backup_broker").await;
    
    let test_order3 = create_test_order("GOOGL", OrderSide::Buy, 10, 2500.00);
    
    // Should failover to tertiary
    let result3 = manager.execute_order_with_failover(&test_order3).await;
    match result3 {
        Ok((broker_name, execution_id)) => {
            assert_eq!(broker_name, "tertiary_broker");
            info!("✅ Second failover used tertiary broker: {}", execution_id);
        }
        Err(e) => {
            panic!("❌ Second failover should succeed: {}", e);
        }
    }
    
    // Simulate all brokers failing
    manager.simulate_broker_failure("tertiary_broker").await;
    
    let test_order4 = create_test_order("TSLA", OrderSide::Sell, 25, 800.00);
    
    // Should fail completely
    let result4 = manager.execute_order_with_failover(&test_order4).await;
    match result4 {
        Ok((broker_name, _)) => {
            panic!("❌ Execution should fail when all brokers are down, but succeeded on: {}", broker_name);
        }
        Err(e) => {
            info!("✅ Properly failed when all brokers down: {}", e);
            assert!(e.contains("All brokers failed"));
        }
    }
    
    // Test broker recovery
    manager.simulate_broker_recovery("backup_broker").await;
    
    let test_order5 = create_test_order("AMZN", OrderSide::Buy, 5, 3000.00);
    
    // Should work again with recovered broker
    let result5 = manager.execute_order_with_failover(&test_order5).await;
    match result5 {
        Ok((broker_name, execution_id)) => {
            assert_eq!(broker_name, "backup_broker");
            info!("✅ Recovery test used recovered broker: {}", execution_id);
        }
        Err(e) => {
            panic!("❌ Recovery execution should succeed: {}", e);
        }
    }
    
    // Verify routing statistics
    let stats = manager.get_routing_statistics().await;
    info!("📊 Final routing statistics:");
    for (broker, count) in stats {
        info!("   {}: {} orders", broker, count);
    }
    
    info!("✅ Basic broker failover test completed");
}

#[tokio::test]
async fn test_latency_based_routing() {
    info!("🔄 Testing latency-based smart routing");
    
    let mut manager = MultiBrokerManager::new(500); // 500ms failover threshold
    
    // Add brokers with different latencies
    manager.add_broker(MockBroker::new("fast_broker", 10));     // 10ms latency
    manager.add_broker(MockBroker::new("medium_broker", 100));  // 100ms latency
    manager.add_broker(MockBroker::new("slow_broker", 400));    // 400ms latency
    
    let iterations = 20;
    let mut execution_counts = HashMap::new();
    
    for i in 0..iterations {
        let test_order = create_test_order("AAPL", OrderSide::Buy, 100, 150.00 + i as f64);
        
        match manager.execute_order_with_failover(&test_order).await {
            Ok((broker_name, _)) => {
                *execution_counts.entry(broker_name).or_insert(0) += 1;
            }
            Err(e) => {
                error!("❌ Order {} failed: {}", i, e);
            }
        }
        
        // Small delay between orders
        tokio::time::sleep(Duration::from_millis(10)).await;
    }
    
    info!("📊 Latency-based routing results:");
    for (broker, count) in &execution_counts {
        info!("   {}: {} orders ({}%)", broker, count, (count * 100) / iterations);
    }
    
    // Fast broker should get most orders (since it becomes primary after first success)
    let fast_count = execution_counts.get("fast_broker").unwrap_or(&0);
    assert!(*fast_count > iterations / 2, 
           "Fast broker should handle majority of orders, got {}/{}", fast_count, iterations);
    
    info!("✅ Latency-based routing test completed");
}

#[tokio::test]
async fn test_broker_health_monitoring() {
    info!("🔄 Testing broker health monitoring");
    
    let mut manager = MultiBrokerManager::new(1000);
    
    // Add brokers
    manager.add_broker(MockBroker::new("healthy_broker", 50));
    manager.add_broker(MockBroker::new("unstable_broker", 100));
    manager.add_broker(MockBroker::new("failing_broker", 150));
    
    // Initial health check - all should be healthy
    let initial_health = manager.get_broker_health_status().await;
    info!("📋 Initial broker health:");
    for (broker, status) in &initial_health {
        info!("   {}: {}", broker, if *status { "HEALTHY" } else { "FAILED" });
        assert!(*status, "All brokers should initially be healthy");
    }
    
    // Simulate different failure scenarios
    manager.simulate_broker_failure("failing_broker").await;
    
    // Set unstable broker to have high failure rate
    if let Some(unstable_broker) = manager.brokers.iter().find(|b| b.name == "unstable_broker") {
        unstable_broker.set_failure_rate(0.7).await; // 70% failure rate
    }
    
    // Test orders with health monitoring
    let test_orders = vec![
        create_test_order("AAPL", OrderSide::Buy, 100, 150.00),
        create_test_order("MSFT", OrderSide::Sell, 50, 300.00),
        create_test_order("GOOGL", OrderSide::Buy, 10, 2500.00),
        create_test_order("TSLA", OrderSide::Sell, 25, 800.00),
        create_test_order("AMZN", OrderSide::Buy, 5, 3000.00),
    ];
    
    let mut successful_executions = 0;
    let mut failed_executions = 0;
    
    for (i, order) in test_orders.iter().enumerate() {
        match manager.execute_order_with_failover(order).await {
            Ok((broker_name, execution_id)) => {
                successful_executions += 1;
                info!("✅ Order {} executed on {}: {}", i, broker_name, execution_id);
                
                // Should not use failing broker
                assert_ne!(broker_name, "failing_broker", 
                          "Should not route to failed broker");
            }
            Err(e) => {
                failed_executions += 1;
                warn!("⚠️  Order {} failed: {}", i, e);
            }
        }
    }
    
    info!("📊 Health monitoring results:");
    info!("   Successful executions: {}", successful_executions);
    info!("   Failed executions: {}", failed_executions);
    
    // Most orders should succeed despite broker failures
    assert!(successful_executions >= 3, 
           "Should have at least 3 successful executions with healthy brokers available");
    
    // Check final health status
    let final_health = manager.get_broker_health_status().await;
    info!("📋 Final broker health:");
    for (broker, status) in &final_health {
        info!("   {}: {}", broker, if *status { "HEALTHY" } else { "FAILED" });
    }
    
    assert!(!final_health["failing_broker"], "Failing broker should be marked as failed");
    assert!(final_health["healthy_broker"], "Healthy broker should remain healthy");
    
    info!("✅ Broker health monitoring test completed");
}

#[tokio::test]
async fn test_load_balancing_across_brokers() {
    info!("🔄 Testing load balancing across multiple brokers");
    
    let mut manager = MultiBrokerManager::new(1000);
    
    // Add multiple healthy brokers with similar latencies
    manager.add_broker(MockBroker::new("broker_a", 50));
    manager.add_broker(MockBroker::new("broker_b", 55));
    manager.add_broker(MockBroker::new("broker_c", 60));
    manager.add_broker(MockBroker::new("broker_d", 65));
    
    let total_orders = 40;
    let mut broker_usage = HashMap::new();
    
    // Execute many orders to test distribution
    for i in 0..total_orders {
        let test_order = create_test_order(
            &format!("STOCK{}", i % 10), 
            if i % 2 == 0 { OrderSide::Buy } else { OrderSide::Sell },
            100 + (i as i64 * 10),
            100.0 + (i as f64 * 0.5)
        );
        
        match manager.execute_order_with_failover(&test_order).await {
            Ok((broker_name, _)) => {
                *broker_usage.entry(broker_name).or_insert(0) += 1;
            }
            Err(e) => {
                error!("❌ Order {} failed: {}", i, e);
            }
        }
        
        // Small delay to allow for realistic order flow
        tokio::time::sleep(Duration::from_millis(5)).await;
    }
    
    info!("📊 Load balancing results:");
    let mut total_executed = 0;
    for (broker, count) in &broker_usage {
        let percentage = (count * 100) / total_orders;
        info!("   {}: {} orders ({}%)", broker, count, percentage);
        total_executed += count;
    }
    
    info!("   Total executed: {}/{}", total_executed, total_orders);
    
    // Should have high success rate
    assert!(total_executed >= (total_orders * 8) / 10, 
           "Should execute at least 80% of orders");
    
    // Note: Since we use failover logic (primary broker preference), 
    // we expect the first successful broker to handle most orders.
    // In a true load balancer, we'd expect more even distribution.
    
    info!("✅ Load balancing test completed");
}

#[tokio::test]
async fn test_real_broker_integration_failover() {
    info!("🔄 Testing failover with real broker configurations");
    
    // Create real broker configurations (will fail gracefully in CI)
    let ib_config = InteractiveBrokersConfig {
        enabled: true,
        host: env::var("FOXHUNT_IB_HOST").unwrap_or_else(|_| "127.0.0.1".to_string()),
        port: 7497,
        client_id: 1,
        account_id: Some("DU123456".to_string()),
        connection_timeout_secs: 5,
        request_timeout_secs: 3,
        heartbeat_interval_secs: 30,
        max_reconnect_attempts: 2,
        paper_trading: true,
    };
    
    let ic_config = ICMarketsConfig {
        enabled: true,
        fix_endpoint: "demo1.p.ctrader.com".to_string(),
        fix_port: 5034,
        sender_comp_id: "FOXHUNT_TEST".to_string(),
        target_comp_id: "ICMARKETS".to_string(),
        rest_base_url: "https://api-demo.ctrader.com".to_string(),
        rate_limit_per_minute: 60,
        username: env::var("FOXHUNT_IC_USERNAME").ok(),
        password: env::var("FOXHUNT_IC_PASSWORD").ok(),
        account_id: env::var("FOXHUNT_IC_ACCOUNT_ID").ok(),
    };
    
    // Test broker creation
    let ib_client = InteractiveBrokersClient::new(ib_config);
    let ic_client = ICMarketsClient::new(ic_config);
    
    // Verify initial states
    assert!(!ib_client.is_connected());
    assert!(!ic_client.is_connected());
    
    info!("✅ Real broker clients created successfully");
    
    // Test connection attempts (will gracefully fail in CI)
    let test_order = create_test_order("AAPL", OrderSide::Buy, 100, 150.50);
    
    // Try IB first
    info!("🔄 Testing IB connection and order submission");
    let ib_order_result = ib_client.submit_order(&test_order).await;
    match ib_order_result {
        Ok(order_id) => {
            info!("✅ IB order submitted successfully: {}", order_id);
            
            // Try to cancel the order
            let cancel_result = ib_client.cancel_order(&order_id).await;
            match cancel_result {
                Ok(()) => info!("✅ IB order cancelled successfully"),
                Err(e) => warn!("⚠️  IB order cancellation failed: {}", e),
            }
        }
        Err(e) => {
            info!("⚠️  IB order failed (expected in CI): {}", e);
            
            // Should contain appropriate error message
            assert!(e.to_string().to_lowercase().contains("not connected") || 
                   e.to_string().to_lowercase().contains("not available"));
        }
    }
    
    // Try ICMarkets as failover
    info!("🔄 Testing ICMarkets as failover broker");
    let ic_order_result = ic_client.submit_order(&test_order).await;
    match ic_order_result {
        Ok(order_id) => {
            info!("✅ ICMarkets order submitted successfully: {}", order_id);
            
            // Try to cancel the order
            let cancel_result = ic_client.cancel_order(&order_id).await;
            match cancel_result {
                Ok(()) => info!("✅ ICMarkets order cancelled successfully"),
                Err(e) => warn!("⚠️  ICMarkets order cancellation failed: {}", e),
            }
        }
        Err(e) => {
            info!("⚠️  ICMarkets order failed (expected in CI): {}", e);
            
            // Should contain appropriate error message
            assert!(e.to_string().to_lowercase().contains("not logged on") || 
                   e.to_string().to_lowercase().contains("not available"));
        }
    }
    
    // Test broker status reporting
    info!("📊 Broker status summary:");
    info!("   IB Connection Status: {:?}", ib_client.connection_status());
    info!("   ICMarkets Connection Status: {:?}", ic_client.connection_status());
    
    // Both should report disconnected status in CI environment
    assert_eq!(ib_client.connection_status(), BrokerConnectionStatus::Disconnected);
    assert_eq!(ic_client.connection_status(), BrokerConnectionStatus::Disconnected);
    
    info!("✅ Real broker integration failover test completed");
}

#[tokio::test]
async fn test_concurrent_broker_operations() {
    info!("🔄 Testing concurrent operations across multiple brokers");
    
    let mut manager = MultiBrokerManager::new(1000);
    
    // Add brokers with different characteristics
    manager.add_broker(MockBroker::new("fast_broker", 20));
    manager.add_broker(MockBroker::new("reliable_broker", 80));
    manager.add_broker(MockBroker::new("capacity_broker", 120));
    
    // Set different failure rates to simulate real-world conditions
    if let Some(fast_broker) = manager.brokers.iter().find(|b| b.name == "fast_broker") {
        fast_broker.set_failure_rate(0.1).await; // 10% failure rate
    }
    if let Some(capacity_broker) = manager.brokers.iter().find(|b| b.name == "capacity_broker") {
        capacity_broker.set_failure_rate(0.05).await; // 5% failure rate
    }
    
    let concurrent_orders = 50;
    let mut handles = Vec::new();
    
    // Launch concurrent order executions
    for i in 0..concurrent_orders {
        let manager_ref = Arc::new(&manager);
        let handle = tokio::spawn(async move {
            let order = create_test_order(
                &format!("STOCK{}", i % 20),
                if i % 2 == 0 { OrderSide::Buy } else { OrderSide::Sell },
                100 + (i as i64 * 5),
                100.0 + (i as f64 * 0.25)
            );
            
            manager_ref.execute_order_with_failover(&order).await
        });
        handles.push(handle);
    }
    
    // Wait for all orders to complete
    let results = futures::future::join_all(handles).await;
    
    let mut successful_orders = 0;
    let mut failed_orders = 0;
    let mut broker_distribution = HashMap::new();
    
    for (i, result) in results.into_iter().enumerate() {
        match result {
            Ok(Ok((broker_name, execution_id))) => {
                successful_orders += 1;
                *broker_distribution.entry(broker_name.clone()).or_insert(0) += 1;
                
                if i < 5 { // Log first few successes
                    info!("✅ Concurrent order {} executed on {}: {}", i, broker_name, execution_id);
                }
            }
            Ok(Err(e)) => {
                failed_orders += 1;
                if failed_orders <= 3 { // Log first few failures
                    warn!("⚠️  Concurrent order {} failed: {}", i, e);
                }
            }
            Err(e) => {
                failed_orders += 1;
                error!("❌ Concurrent task {} panicked: {}", i, e);
            }
        }
    }
    
    info!("📊 Concurrent operations results:");
    info!("   Total orders: {}", concurrent_orders);
    info!("   Successful: {} ({}%)", successful_orders, (successful_orders * 100) / concurrent_orders);
    info!("   Failed: {} ({}%)", failed_orders, (failed_orders * 100) / concurrent_orders);
    
    info!("📊 Broker distribution:");
    for (broker, count) in broker_distribution {
        let percentage = (count * 100) / successful_orders.max(1);
        info!("   {}: {} orders ({}%)", broker, count, percentage);
    }
    
    // Should have high success rate even with concurrent operations
    assert!(successful_orders >= (concurrent_orders * 8) / 10,
           "Should handle at least 80% of concurrent orders successfully");
    
    // Verify final broker statistics
    let final_stats = manager.get_routing_statistics().await;
    info!("📊 Final routing statistics:");
    for (broker, count) in final_stats {
        info!("   {}: {} total orders", broker, count);
    }
    
    info!("✅ Concurrent broker operations test completed");
}

#[tokio::test]
async fn test_broker_recovery_scenarios() {
    info!("🔄 Testing broker recovery scenarios");
    
    let mut manager = MultiBrokerManager::new(500);
    
    // Add brokers
    manager.add_broker(MockBroker::new("primary_broker", 50));
    manager.add_broker(MockBroker::new("secondary_broker", 100));
    
    // Normal operation
    let order1 = create_test_order("AAPL", OrderSide::Buy, 100, 150.00);
    let result1 = manager.execute_order_with_failover(&order1).await;
    assert!(result1.is_ok());
    info!("✅ Normal operation works");
    
    // Simulate primary failure
    manager.simulate_broker_failure("primary_broker").await;
    tokio::time::sleep(Duration::from_millis(100)).await;
    
    let order2 = create_test_order("MSFT", OrderSide::Sell, 50, 300.00);
    let result2 = manager.execute_order_with_failover(&order2).await;
    match result2 {
        Ok((broker_name, _)) => {
            assert_eq!(broker_name, "secondary_broker");
            info!("✅ Failover to secondary broker works");
        }
        Err(e) => panic!("❌ Failover should succeed: {}", e),
    }
    
    // Simulate primary recovery
    manager.simulate_broker_recovery("primary_broker").await;
    tokio::time::sleep(Duration::from_millis(100)).await;
    
    // Test that primary becomes available again
    let order3 = create_test_order("GOOGL", OrderSide::Buy, 10, 2500.00);
    let result3 = manager.execute_order_with_failover(&order3).await;
    match result3 {
        Ok((broker_name, _)) => {
            // Should now prefer the secondary broker (which became primary after failover)
            // or could be primary if routing logic prefers recovered brokers
            info!("✅ Order executed on broker: {}", broker_name);
        }
        Err(e) => panic!("❌ Recovery execution should succeed: {}", e),
    }
    
    // Test rapid failure/recovery cycles
    for cycle in 1..=3 {
        info!("🔄 Testing failure/recovery cycle {}", cycle);
        
        manager.simulate_broker_failure("primary_broker").await;
        tokio::time::sleep(Duration::from_millis(50)).await;
        
        let cycle_order = create_test_order("TSLA", OrderSide::Sell, 25, 800.00);
        let cycle_result = manager.execute_order_with_failover(&cycle_order).await;
        assert!(cycle_result.is_ok(), "Order should succeed during cycle {}", cycle);
        
        manager.simulate_broker_recovery("primary_broker").await;
        tokio::time::sleep(Duration::from_millis(50)).await;
    }
    
    // Verify system stability after rapid cycles
    let final_order = create_test_order("AMZN", OrderSide::Buy, 5, 3000.00);
    let final_result = manager.execute_order_with_failover(&final_order).await;
    assert!(final_result.is_ok(), "System should be stable after rapid cycles");
    
    // Check final health status
    let health_status = manager.get_broker_health_status().await;
    info!("📋 Final health status after recovery testing:");
    for (broker, status) in health_status {
        info!("   {}: {}", broker, if status { "HEALTHY" } else { "FAILED" });
    }
    
    info!("✅ Broker recovery scenarios test completed");
}