//! Integration tests for Rollback Automation //! //! Tests the complete rollback automation system with actual execution logic //! for all 4 failure scenarios: //! 1. DailyLossExceeded (>$2K loss) //! 2. HighDisagreement (>70% for 1 hour) //! 3. ModelFailure (>3 consecutive errors) //! 4. CascadeFailure (2+ models fail) use std::time::Duration; use trading_service::rollback_automation::{ RollbackAction, RollbackAutomation, RollbackConfig, RollbackScenario, }; #[tokio::test] async fn test_rollback_automation_creation_with_dependencies() { let config = RollbackConfig::default(); let automation = RollbackAutomation::new(config).with_account_id("TEST_ACCOUNT".to_string()); // Verify initial state assert!(automation.is_trading_enabled()); assert!(!automation.is_trading_halted().await); let state = automation.get_state().await; assert_eq!(state.daily_pnl_usd, 0.0); assert!(state.active_scenarios.is_empty()); assert!(!state.trading_halted); } #[tokio::test] async fn test_emergency_halt_execution() { let config = RollbackConfig::default(); let automation = RollbackAutomation::new(config); // Initially trading should be enabled assert!(automation.is_trading_enabled()); // Trigger daily loss scenario automation .trigger_scenario_manual(RollbackScenario::DailyLossExceeded) .await .unwrap(); // Start monitoring to execute actions let mut automation = automation; automation.start_monitoring().await.unwrap(); // Wait for action execution tokio::time::sleep(Duration::from_millis(500)).await; // Verify emergency halt was executed assert!(!automation.is_trading_enabled()); assert!(automation.is_trading_halted().await); let state = automation.get_state().await; assert!(state .executed_actions .iter() .any(|(a, _)| matches!(a, RollbackAction::EmergencyHalt))); automation.stop_monitoring().await; } #[tokio::test] async fn test_position_reduction_execution() { let config = RollbackConfig { position_reduction_factor: 0.5, // 50% reduction ..Default::default() }; let automation = RollbackAutomation::new(config); // Trigger high disagreement scenario automation .trigger_scenario_manual(RollbackScenario::HighDisagreement) .await .unwrap(); // Start monitoring let mut automation = automation; automation.start_monitoring().await.unwrap(); // Wait for action execution tokio::time::sleep(Duration::from_millis(500)).await; // Verify position reduction was attempted let state = automation.get_state().await; assert!(state .executed_actions .iter() .any(|(a, _)| matches!(a, RollbackAction::ReducePositions))); automation.stop_monitoring().await; } #[tokio::test] async fn test_model_disabling_confirmation() { let config = RollbackConfig::default(); let automation = RollbackAutomation::new(config); // Trigger model failure scenario automation .trigger_scenario_manual(RollbackScenario::ModelFailure) .await .unwrap(); // Start monitoring let mut automation = automation; automation.start_monitoring().await.unwrap(); // Wait for action execution tokio::time::sleep(Duration::from_millis(500)).await; // Verify model disabling was confirmed let state = automation.get_state().await; assert!(state .executed_actions .iter() .any(|(a, _)| matches!(a, RollbackAction::DisableModels))); automation.stop_monitoring().await; } #[tokio::test] async fn test_baseline_revert_execution() { let config = RollbackConfig::default(); let automation = RollbackAutomation::new(config); // Trigger cascade failure scenario automation .trigger_scenario_manual(RollbackScenario::CascadeFailure) .await .unwrap(); // Start monitoring let mut automation = automation; automation.start_monitoring().await.unwrap(); // Wait for action execution tokio::time::sleep(Duration::from_millis(500)).await; // Verify baseline revert was attempted let state = automation.get_state().await; assert!(state .executed_actions .iter() .any(|(a, _)| matches!(a, RollbackAction::RevertToBaseline))); automation.stop_monitoring().await; } #[tokio::test] async fn test_daily_loss_scenario_full_recovery() { let config = RollbackConfig { daily_loss_threshold_usd: 2000.0, enable_automatic_rollback: true, ..Default::default() }; let automation = RollbackAutomation::new(config); // Simulate large daily loss automation.update_daily_pnl(-2500.0).await.unwrap(); // Start monitoring let mut automation = automation; automation.start_monitoring().await.unwrap(); // Wait for recovery tokio::time::sleep(Duration::from_secs(2)).await; // Verify complete recovery let state = automation.get_state().await; // Should trigger DailyLossExceeded scenario assert!(state .active_scenarios .contains_key(&RollbackScenario::DailyLossExceeded)); // Should execute EmergencyHalt and ReducePositions assert!(state .executed_actions .iter() .any(|(a, _)| matches!(a, RollbackAction::EmergencyHalt))); assert!(state .executed_actions .iter() .any(|(a, _)| matches!(a, RollbackAction::ReducePositions))); // Trading should be halted assert!(!automation.is_trading_enabled()); // Recovery should have started assert!(state.recovery_start.is_some()); automation.stop_monitoring().await; } #[tokio::test] async fn test_high_disagreement_scenario_full_recovery() { let config = RollbackConfig { high_disagreement_threshold: 0.70, disagreement_duration_secs: 10, // 10 seconds for testing monitoring_interval_secs: 1, enable_automatic_rollback: true, ..Default::default() }; let automation = RollbackAutomation::new(config); // Record sustained high disagreement for _ in 0..15 { automation.record_disagreement(0.75).await.unwrap(); tokio::time::sleep(Duration::from_millis(100)).await; } // Start monitoring let mut automation = automation; automation.start_monitoring().await.unwrap(); // Wait for recovery tokio::time::sleep(Duration::from_secs(2)).await; // Verify recovery actions let state = automation.get_state().await; // Should execute RevertToBaseline and ReducePositions assert!(state .executed_actions .iter() .any(|(a, _)| matches!(a, RollbackAction::RevertToBaseline))); assert!(state .executed_actions .iter() .any(|(a, _)| matches!(a, RollbackAction::ReducePositions))); automation.stop_monitoring().await; } #[tokio::test] async fn test_cascade_failure_scenario_full_recovery() { let config = RollbackConfig { cascade_failure_threshold: 2, enable_automatic_rollback: true, ..Default::default() }; let automation = RollbackAutomation::new(config); // Trigger cascade failure automation .trigger_scenario_manual(RollbackScenario::CascadeFailure) .await .unwrap(); // Start monitoring let mut automation = automation; automation.start_monitoring().await.unwrap(); // Wait for recovery tokio::time::sleep(Duration::from_secs(2)).await; // Verify emergency response let state = automation.get_state().await; // Should execute EmergencyHalt and RevertToBaseline assert!(state .executed_actions .iter() .any(|(a, _)| matches!(a, RollbackAction::EmergencyHalt))); assert!(state .executed_actions .iter() .any(|(a, _)| matches!(a, RollbackAction::RevertToBaseline))); // Trading should be halted assert!(!automation.is_trading_enabled()); automation.stop_monitoring().await; } #[tokio::test] async fn test_recovery_duration_tracking() { let config = RollbackConfig::default(); let automation = RollbackAutomation::new(config); // Trigger scenario automation .trigger_scenario_manual(RollbackScenario::DailyLossExceeded) .await .unwrap(); // Start monitoring let mut automation = automation; automation.start_monitoring().await.unwrap(); // Wait for recovery tokio::time::sleep(Duration::from_millis(500)).await; // Verify recovery duration is tracked let duration = automation.get_recovery_duration().await; assert!(duration.is_some()); assert!(duration.unwrap().as_millis() >= 100); assert!(duration.unwrap() < Duration::from_secs(300)); // <5 minutes automation.stop_monitoring().await; } #[tokio::test] async fn test_rollback_report_generation() { let config = RollbackConfig::default(); let automation = RollbackAutomation::new(config); // Trigger multiple scenarios automation .trigger_scenario_manual(RollbackScenario::DailyLossExceeded) .await .unwrap(); automation .trigger_scenario_manual(RollbackScenario::ModelFailure) .await .unwrap(); // Start monitoring let mut automation = automation; automation.start_monitoring().await.unwrap(); // Wait for recovery tokio::time::sleep(Duration::from_secs(1)).await; // Generate report let state = automation.get_state().await; let report = trading_service::rollback_automation::RollbackReport::from_state(&state); // Verify report content assert!(!report.scenarios_triggered.is_empty()); assert!(!report.actions_executed.is_empty()); assert!(report.recovery_duration.is_some()); assert!(report.trading_halted); automation.stop_monitoring().await; } #[tokio::test] async fn test_automatic_vs_manual_rollback() { // Test with automatic rollback disabled let config_manual = RollbackConfig { enable_automatic_rollback: false, ..Default::default() }; let automation_manual = RollbackAutomation::new(config_manual); automation_manual .trigger_scenario_manual(RollbackScenario::DailyLossExceeded) .await .unwrap(); let mut automation_manual = automation_manual; automation_manual.start_monitoring().await.unwrap(); tokio::time::sleep(Duration::from_millis(500)).await; // Should detect but not execute let state_manual = automation_manual.get_state().await; assert!(!state_manual.active_scenarios.is_empty()); assert!(state_manual.executed_actions.is_empty()); // No actions when disabled automation_manual.stop_monitoring().await; // Test with automatic rollback enabled let config_auto = RollbackConfig { enable_automatic_rollback: true, ..Default::default() }; let automation_auto = RollbackAutomation::new(config_auto); automation_auto .trigger_scenario_manual(RollbackScenario::DailyLossExceeded) .await .unwrap(); let mut automation_auto = automation_auto; automation_auto.start_monitoring().await.unwrap(); tokio::time::sleep(Duration::from_millis(500)).await; // Should detect AND execute let state_auto = automation_auto.get_state().await; assert!(!state_auto.active_scenarios.is_empty()); assert!(!state_auto.executed_actions.is_empty()); // Actions executed automation_auto.stop_monitoring().await; } #[tokio::test] async fn test_reset_functionality() { let config = RollbackConfig::default(); let automation = RollbackAutomation::new(config); // Trigger scenarios and execute actions automation.update_daily_pnl(-3000.0).await.unwrap(); automation .trigger_scenario_manual(RollbackScenario::DailyLossExceeded) .await .unwrap(); let mut automation = automation; automation.start_monitoring().await.unwrap(); tokio::time::sleep(Duration::from_millis(500)).await; // Verify state is dirty let state_before = automation.get_state().await; assert!(!state_before.active_scenarios.is_empty()); assert!(!state_before.executed_actions.is_empty()); // Reset automation.reset_all().await.unwrap(); // Verify state is clean let state_after = automation.get_state().await; assert_eq!(state_after.daily_pnl_usd, 0.0); assert!(state_after.active_scenarios.is_empty()); assert!(state_after.executed_actions.is_empty()); assert!(!state_after.trading_halted); automation.stop_monitoring().await; }