#![allow(unexpected_cfgs)] #![cfg(feature = "__trading_service_integration")] //! Comprehensive Integration Tests for Rollback Automation //! //! Tests all 4 failure scenarios with automatic recovery: //! 1. Daily loss exceeds $2K //! 2. Model disagreement >70% for 1 hour //! 3. Single model >3 consecutive errors //! 4. Cascade failure (2+ models fail) use std::sync::Arc; use std::time::Duration; use tokio::time::sleep; use trading_service::ensemble_coordinator::EnsembleCoordinator; use trading_service::ensemble_risk_manager::{EnsembleRiskConfig, EnsembleRiskManager}; use trading_service::rollback_automation::{ RollbackAction, RollbackAutomation, RollbackConfig, RollbackReport, RollbackScenario, }; /// Test helper to create automation with default config fn create_automation() -> RollbackAutomation { let config = RollbackConfig { daily_loss_threshold_usd: 2000.0, high_disagreement_threshold: 0.70, disagreement_duration_secs: 10, // 10 seconds for testing max_consecutive_errors: 3, cascade_failure_threshold: 2, position_reduction_factor: 0.50, monitoring_interval_secs: 1, // 1 second for testing recovery_timeout_secs: 300, enable_automatic_rollback: true, }; RollbackAutomation::new(config) } /// Test helper to create ensemble risk manager async fn create_risk_manager() -> Arc { let config = EnsembleRiskConfig { max_consecutive_errors: 3, cascade_failure_threshold: 2, cascade_detection_window_secs: 60, ..Default::default() }; let manager = EnsembleRiskManager::new(config); // Register test models manager.register_model("DQN".to_string()).await.unwrap(); manager.register_model("PPO".to_string()).await.unwrap(); manager.register_model("TFT".to_string()).await.unwrap(); Arc::new(manager) } /// Test helper to create ensemble coordinator async fn create_coordinator() -> Arc { let coordinator = EnsembleCoordinator::new(); coordinator .register_model("DQN".to_string(), 0.33) .await .unwrap(); coordinator .register_model("PPO".to_string(), 0.33) .await .unwrap(); coordinator .register_model("TFT".to_string(), 0.34) .await .unwrap(); Arc::new(coordinator) } // ============================================================================ // SCENARIO 1: Daily Loss Exceeds $2K // ============================================================================ #[tokio::test] async fn test_scenario_1_daily_loss_exceeded_basic() { let automation = create_automation(); // Simulate daily loss exceeding threshold automation.update_daily_pnl(-2500.0).await.unwrap(); // Trigger scenario manually automation .trigger_scenario_manual(RollbackScenario::DailyLossExceeded) .await .unwrap(); let state = automation.get_state().await; assert!(state .active_scenarios .contains_key(&RollbackScenario::DailyLossExceeded)); assert_eq!(state.daily_pnl_usd, -2500.0); } #[tokio::test] async fn test_scenario_1_emergency_halt_executed() { let automation = create_automation(); // Trigger scenario automation .trigger_scenario_manual(RollbackScenario::DailyLossExceeded) .await .unwrap(); // Execute recovery let config = automation.config.clone(); RollbackAutomation::execute_recovery_actions(&config, &automation.state) .await .unwrap(); // Verify emergency halt assert!(automation.is_trading_halted().await); let state = automation.get_state().await; assert!(state .executed_actions .iter() .any(|(a, _)| *a == RollbackAction::EmergencyHalt)); } #[tokio::test] async fn test_scenario_1_position_reduction_executed() { let automation = create_automation(); // Trigger scenario automation .trigger_scenario_manual(RollbackScenario::DailyLossExceeded) .await .unwrap(); // Execute recovery let config = automation.config.clone(); RollbackAutomation::execute_recovery_actions(&config, &automation.state) .await .unwrap(); // Verify position reduction assert!(automation.are_positions_reduced().await); let state = automation.get_state().await; assert!(state .executed_actions .iter() .any(|(a, _)| *a == RollbackAction::ReducePositions)); } #[tokio::test] async fn test_scenario_1_recovery_time_under_5_minutes() { let automation = create_automation(); // Trigger and recover automation .trigger_scenario_manual(RollbackScenario::DailyLossExceeded) .await .unwrap(); let config = automation.config.clone(); RollbackAutomation::execute_recovery_actions(&config, &automation.state) .await .unwrap(); // Check recovery time let duration = automation.get_recovery_duration().await; assert!(duration.is_some()); assert!(duration.unwrap().as_secs() < 300); // Less than 5 minutes } #[tokio::test] async fn test_scenario_1_full_recovery_sequence() { let automation = create_automation(); // Simulate loss automation.update_daily_pnl(-2500.0).await.unwrap(); // Trigger automation .trigger_scenario_manual(RollbackScenario::DailyLossExceeded) .await .unwrap(); // Execute recovery let config = automation.config.clone(); RollbackAutomation::execute_recovery_actions(&config, &automation.state) .await .unwrap(); // Verify complete recovery let state = automation.get_state().await; let report = RollbackReport::from_state(&state); assert!(report.trading_halted); assert!(report.positions_reduced); assert!(!report.actions_executed.is_empty()); } // ============================================================================ // SCENARIO 2: Model Disagreement >70% for 1 Hour // ============================================================================ #[tokio::test] async fn test_scenario_2_high_disagreement_detection() { let automation = create_automation(); // Record sustained high disagreement for _ in 0..15 { automation.record_disagreement(0.75).await.unwrap(); sleep(Duration::from_millis(100)).await; } // Trigger scenario automation .trigger_scenario_manual(RollbackScenario::HighDisagreement) .await .unwrap(); let state = automation.get_state().await; assert!(state .active_scenarios .contains_key(&RollbackScenario::HighDisagreement)); } #[tokio::test] async fn test_scenario_2_baseline_revert_executed() { let automation = create_automation(); // Trigger scenario automation .trigger_scenario_manual(RollbackScenario::HighDisagreement) .await .unwrap(); // Execute recovery let config = automation.config.clone(); RollbackAutomation::execute_recovery_actions(&config, &automation.state) .await .unwrap(); // Verify baseline mode assert!(automation.is_baseline_mode_active().await); let state = automation.get_state().await; assert!(state .executed_actions .iter() .any(|(a, _)| *a == RollbackAction::RevertToBaseline)); } #[tokio::test] async fn test_scenario_2_position_reduction() { let automation = create_automation(); // Trigger scenario automation .trigger_scenario_manual(RollbackScenario::HighDisagreement) .await .unwrap(); // Execute recovery let config = automation.config.clone(); RollbackAutomation::execute_recovery_actions(&config, &automation.state) .await .unwrap(); // Verify position reduction assert!(automation.are_positions_reduced().await); } #[tokio::test] async fn test_scenario_2_disagreement_windowing() { let automation = create_automation(); // Record disagreement within window for _ in 0..10 { automation.record_disagreement(0.75).await.unwrap(); } let state = automation.get_state().await; assert!(!state.disagreement_history.is_empty()); assert!(state.disagreement_history.len() <= 10); } #[tokio::test] async fn test_scenario_2_recovery_time() { let automation = create_automation(); // Trigger and recover automation .trigger_scenario_manual(RollbackScenario::HighDisagreement) .await .unwrap(); let config = automation.config.clone(); RollbackAutomation::execute_recovery_actions(&config, &automation.state) .await .unwrap(); // Check recovery time let duration = automation.get_recovery_duration().await; assert!(duration.is_some()); assert!(duration.unwrap().as_secs() < 300); } // ============================================================================ // SCENARIO 3: Single Model >3 Consecutive Errors // ============================================================================ #[tokio::test] async fn test_scenario_3_model_failure_detection() { let risk_manager = create_risk_manager().await; // Simulate 3 consecutive errors for DQN for _ in 0..3 { risk_manager .record_prediction_result("DQN", false) .await .unwrap(); } // Check model health let health = risk_manager.get_model_health("DQN").await.unwrap(); assert!(!health.enabled); assert_eq!(health.consecutive_errors, 3); } #[tokio::test] async fn test_scenario_3_model_disabled() { let risk_manager = create_risk_manager().await; let automation = create_automation().with_ensemble_risk_manager(Arc::clone(&risk_manager)); // Fail DQN model for _ in 0..3 { risk_manager .record_prediction_result("DQN", false) .await .unwrap(); } // Trigger scenario automation .trigger_scenario_manual(RollbackScenario::ModelFailure) .await .unwrap(); // Execute recovery let config = automation.config.clone(); RollbackAutomation::execute_recovery_actions(&config, &automation.state) .await .unwrap(); let state = automation.get_state().await; assert!(state .executed_actions .iter() .any(|(a, _)| *a == RollbackAction::DisableModels)); } #[tokio::test] async fn test_scenario_3_baseline_mode_activated() { let risk_manager = create_risk_manager().await; let automation = create_automation().with_ensemble_risk_manager(Arc::clone(&risk_manager)); // Trigger model failure automation .trigger_scenario_manual(RollbackScenario::ModelFailure) .await .unwrap(); // Execute recovery let config = automation.config.clone(); RollbackAutomation::execute_recovery_actions(&config, &automation.state) .await .unwrap(); // Verify baseline mode assert!(automation.is_baseline_mode_active().await); } #[tokio::test] async fn test_scenario_3_successful_prediction_resets_errors() { let risk_manager = create_risk_manager().await; // Record 2 errors risk_manager .record_prediction_result("DQN", false) .await .unwrap(); risk_manager .record_prediction_result("DQN", false) .await .unwrap(); // Record success (should reset counter) risk_manager .record_prediction_result("DQN", true) .await .unwrap(); let health = risk_manager.get_model_health("DQN").await.unwrap(); assert!(health.enabled); assert_eq!(health.consecutive_errors, 0); } #[tokio::test] async fn test_scenario_3_recovery_time() { let risk_manager = create_risk_manager().await; let automation = create_automation().with_ensemble_risk_manager(Arc::clone(&risk_manager)); // Trigger and recover automation .trigger_scenario_manual(RollbackScenario::ModelFailure) .await .unwrap(); let config = automation.config.clone(); RollbackAutomation::execute_recovery_actions(&config, &automation.state) .await .unwrap(); let duration = automation.get_recovery_duration().await; assert!(duration.is_some()); assert!(duration.unwrap().as_secs() < 300); } // ============================================================================ // SCENARIO 4: Cascade Failure (2+ Models Fail) // ============================================================================ #[tokio::test] async fn test_scenario_4_cascade_failure_detection() { let risk_manager = create_risk_manager().await; // Fail DQN for _ in 0..3 { risk_manager .record_prediction_result("DQN", false) .await .unwrap(); } // Fail PPO for _ in 0..3 { risk_manager .record_prediction_result("PPO", false) .await .unwrap(); } // Check cascade state let cascade_state = risk_manager.get_cascade_state().await; assert!(cascade_state.is_cascading); assert_eq!(cascade_state.failed_models.len(), 2); } #[tokio::test] async fn test_scenario_4_emergency_halt() { let risk_manager = create_risk_manager().await; let automation = create_automation().with_ensemble_risk_manager(Arc::clone(&risk_manager)); // Trigger cascade failure automation .trigger_scenario_manual(RollbackScenario::CascadeFailure) .await .unwrap(); // Execute recovery let config = automation.config.clone(); RollbackAutomation::execute_recovery_actions(&config, &automation.state) .await .unwrap(); // Verify emergency halt assert!(automation.is_trading_halted().await); } #[tokio::test] async fn test_scenario_4_baseline_revert() { let risk_manager = create_risk_manager().await; let automation = create_automation().with_ensemble_risk_manager(Arc::clone(&risk_manager)); // Trigger cascade failure automation .trigger_scenario_manual(RollbackScenario::CascadeFailure) .await .unwrap(); // Execute recovery let config = automation.config.clone(); RollbackAutomation::execute_recovery_actions(&config, &automation.state) .await .unwrap(); // Verify baseline mode assert!(automation.is_baseline_mode_active().await); } #[tokio::test] async fn test_scenario_4_cascade_within_window() { let risk_manager = create_risk_manager().await; // Fail models within detection window for _ in 0..3 { risk_manager .record_prediction_result("DQN", false) .await .unwrap(); } sleep(Duration::from_millis(100)).await; for _ in 0..3 { risk_manager .record_prediction_result("PPO", false) .await .unwrap(); } let cascade_state = risk_manager.get_cascade_state().await; assert!(cascade_state.is_cascading); } #[tokio::test] async fn test_scenario_4_recovery_time() { let risk_manager = create_risk_manager().await; let automation = create_automation().with_ensemble_risk_manager(Arc::clone(&risk_manager)); // Trigger and recover automation .trigger_scenario_manual(RollbackScenario::CascadeFailure) .await .unwrap(); let config = automation.config.clone(); RollbackAutomation::execute_recovery_actions(&config, &automation.state) .await .unwrap(); let duration = automation.get_recovery_duration().await; assert!(duration.is_some()); assert!(duration.unwrap().as_secs() < 300); } // ============================================================================ // COMPREHENSIVE INTEGRATION TESTS // ============================================================================ #[tokio::test] async fn test_all_scenarios_sequential() { let risk_manager = create_risk_manager().await; let automation = create_automation().with_ensemble_risk_manager(Arc::clone(&risk_manager)); // Scenario 1: Daily loss automation.update_daily_pnl(-2500.0).await.unwrap(); automation .trigger_scenario_manual(RollbackScenario::DailyLossExceeded) .await .unwrap(); // Scenario 2: High disagreement for _ in 0..10 { automation.record_disagreement(0.75).await.unwrap(); } automation .trigger_scenario_manual(RollbackScenario::HighDisagreement) .await .unwrap(); // Scenario 3: Model failure automation .trigger_scenario_manual(RollbackScenario::ModelFailure) .await .unwrap(); // Scenario 4: Cascade failure automation .trigger_scenario_manual(RollbackScenario::CascadeFailure) .await .unwrap(); // Execute recovery let config = automation.config.clone(); RollbackAutomation::execute_recovery_actions(&config, &automation.state) .await .unwrap(); // Verify all actions executed let state = automation.get_state().await; assert!(state.trading_halted); assert!(state.positions_reduced); assert!(state.baseline_mode_active); assert_eq!(state.active_scenarios.len(), 4); } #[tokio::test] async fn test_recovery_report_generation() { let automation = create_automation(); // Trigger scenarios automation .trigger_scenario_manual(RollbackScenario::DailyLossExceeded) .await .unwrap(); automation .trigger_scenario_manual(RollbackScenario::HighDisagreement) .await .unwrap(); // Execute recovery let config = automation.config.clone(); RollbackAutomation::execute_recovery_actions(&config, &automation.state) .await .unwrap(); // Generate report let state = automation.get_state().await; let report = RollbackReport::from_state(&state); assert_eq!(report.scenarios_triggered.len(), 2); assert!(!report.actions_executed.is_empty()); assert!(report.recovery_duration.is_some()); } #[tokio::test] async fn test_success_criteria_validation() { let automation = create_automation(); // Trigger and recover quickly automation .trigger_scenario_manual(RollbackScenario::DailyLossExceeded) .await .unwrap(); let config = automation.config.clone(); RollbackAutomation::execute_recovery_actions(&config, &automation.state) .await .unwrap(); // Wait briefly sleep(Duration::from_millis(100)).await; // Complete recovery let state = automation.get_state().await; let report = RollbackReport::from_state(&state); // Verify meets success criteria if recovery completed if report.recovery_completed { assert!(report.meets_success_criteria()); } } #[tokio::test] async fn test_action_priority_execution() { let automation = create_automation(); // Trigger cascade (should execute EmergencyHalt first) automation .trigger_scenario_manual(RollbackScenario::CascadeFailure) .await .unwrap(); let config = automation.config.clone(); RollbackAutomation::execute_recovery_actions(&config, &automation.state) .await .unwrap(); let state = automation.get_state().await; // Find EmergencyHalt action let halt_idx = state .executed_actions .iter() .position(|(a, _)| *a == RollbackAction::EmergencyHalt); // Find other actions let baseline_idx = state .executed_actions .iter() .position(|(a, _)| *a == RollbackAction::RevertToBaseline); // EmergencyHalt should execute before or at same time as baseline if let (Some(halt), Some(baseline)) = (halt_idx, baseline_idx) { assert!(halt <= baseline); } } #[tokio::test] async fn test_idempotent_action_execution() { let automation = create_automation(); // Trigger scenario automation .trigger_scenario_manual(RollbackScenario::DailyLossExceeded) .await .unwrap(); // Execute recovery multiple times let config = automation.config.clone(); for _ in 0..3 { RollbackAutomation::execute_recovery_actions(&config, &automation.state) .await .unwrap(); } let state = automation.get_state().await; // Each action should only be executed once let halt_count = state .executed_actions .iter() .filter(|(a, _)| *a == RollbackAction::EmergencyHalt) .count(); assert_eq!(halt_count, 1); } #[tokio::test] async fn test_reset_functionality() { let automation = create_automation(); // Trigger scenarios and recover automation .trigger_scenario_manual(RollbackScenario::DailyLossExceeded) .await .unwrap(); automation.update_daily_pnl(-3000.0).await.unwrap(); let config = automation.config.clone(); RollbackAutomation::execute_recovery_actions(&config, &automation.state) .await .unwrap(); // Reset automation.reset_all().await.unwrap(); // Verify clean state let state = automation.get_state().await; assert_eq!(state.daily_pnl_usd, 0.0); assert!(state.active_scenarios.is_empty()); assert!(state.executed_actions.is_empty()); assert!(!state.trading_halted); assert!(!state.positions_reduced); assert!(!state.baseline_mode_active); } #[tokio::test] async fn test_disabled_automatic_rollback() { let config = RollbackConfig { enable_automatic_rollback: false, ..Default::default() }; let automation = RollbackAutomation::new(config.clone()); // Trigger scenario automation .trigger_scenario_manual(RollbackScenario::DailyLossExceeded) .await .unwrap(); // Execute recovery (should do nothing) RollbackAutomation::execute_recovery_actions(&config, &automation.state) .await .unwrap(); let state = automation.get_state().await; assert!(state.executed_actions.is_empty()); assert!(!state.trading_halted); } #[tokio::test] async fn test_daily_reset() { let automation = create_automation(); // Set P&L automation.update_daily_pnl(-1500.0).await.unwrap(); // Reset daily automation.reset_daily().await.unwrap(); let state = automation.get_state().await; assert_eq!(state.daily_pnl_usd, 0.0); } #[tokio::test] async fn test_recovery_timeout_detection() { let config = RollbackConfig { recovery_timeout_secs: 1, // 1 second timeout ..Default::default() }; let automation = RollbackAutomation::new(config.clone()); // Trigger scenario (starts recovery) automation .trigger_scenario_manual(RollbackScenario::DailyLossExceeded) .await .unwrap(); RollbackAutomation::execute_recovery_actions(&config, &automation.state) .await .unwrap(); // Wait for timeout sleep(Duration::from_secs(2)).await; // Check timeout (would be logged as error) let duration = automation.get_recovery_duration().await; assert!(duration.is_some()); assert!(duration.unwrap().as_secs() >= 1); } // ============================================================================ // STRESS TESTS // ============================================================================ #[tokio::test] async fn test_rapid_scenario_triggers() { let automation = create_automation(); // Rapidly trigger scenarios for _ in 0..10 { automation .trigger_scenario_manual(RollbackScenario::DailyLossExceeded) .await .unwrap(); sleep(Duration::from_millis(10)).await; } let state = automation.get_state().await; assert!(state .active_scenarios .contains_key(&RollbackScenario::DailyLossExceeded)); } #[tokio::test] async fn test_concurrent_disagreement_recording() { let automation = Arc::new(create_automation()); // Spawn concurrent tasks recording disagreement let mut handles = vec![]; for i in 0..10 { let auto = Arc::clone(&automation); let handle = tokio::spawn(async move { auto.record_disagreement(0.75 + (i as f64 * 0.01)) .await .unwrap(); }); handles.push(handle); } // Wait for all tasks for handle in handles { handle.await.unwrap(); } let state = automation.get_state().await; assert!(!state.disagreement_history.is_empty()); } #[tokio::test] async fn test_high_frequency_pnl_updates() { let automation = create_automation(); // Rapidly update P&L for i in 0..100 { let pnl = -1000.0 - (i as f64 * 10.0); automation.update_daily_pnl(pnl).await.unwrap(); } let state = automation.get_state().await; assert!(state.daily_pnl_usd < -2000.0); }