foxhunt/services/trading_service/tests/rollback_automation_integration_tests.rs

//! Integration tests for Rollback Automation
//!
//! Tests the complete rollback automation system with actual execution logic
//! for all 4 failure scenarios:
//! 1. DailyLossExceeded (>$2K loss)
//! 2. HighDisagreement (>70% for 1 hour)
//! 3. ModelFailure (>3 consecutive errors)
//! 4. CascadeFailure (2+ models fail)

use std::sync::Arc;
use std::time::Duration;
use tokio;
use trading_service::rollback_automation::{
    RollbackAutomation, RollbackConfig, RollbackScenario, RollbackAction,
};

#[tokio::test]
async fn test_rollback_automation_creation_with_dependencies() {
    let config = RollbackConfig::default();
    let automation = RollbackAutomation::new(config)
        .with_account_id("TEST_ACCOUNT".to_string());

    // Verify initial state
    assert!(automation.is_trading_enabled());
    assert!(!automation.is_trading_halted().await);
    
    let state = automation.get_state().await;
    assert_eq!(state.daily_pnl_usd, 0.0);
    assert!(state.active_scenarios.is_empty());
    assert!(!state.trading_halted);
}

#[tokio::test]
async fn test_emergency_halt_execution() {
    let config = RollbackConfig::default();
    let automation = RollbackAutomation::new(config);

    // Initially trading should be enabled
    assert!(automation.is_trading_enabled());

    // Trigger daily loss scenario
    automation.trigger_scenario_manual(RollbackScenario::DailyLossExceeded).await.unwrap();

    // Start monitoring to execute actions
    let mut automation = automation;
    automation.start_monitoring().await.unwrap();

    // Wait for action execution
    tokio::time::sleep(Duration::from_millis(500)).await;

    // Verify emergency halt was executed
    assert!(!automation.is_trading_enabled());
    assert!(automation.is_trading_halted().await);

    let state = automation.get_state().await;
    assert!(state.executed_actions.iter().any(|(a, _)| matches!(a, RollbackAction::EmergencyHalt)));

    automation.stop_monitoring().await;
}

#[tokio::test]
async fn test_position_reduction_execution() {
    let config = RollbackConfig {
        position_reduction_factor: 0.5, // 50% reduction
        ..Default::default()
    };
    let automation = RollbackAutomation::new(config);

    // Trigger high disagreement scenario
    automation.trigger_scenario_manual(RollbackScenario::HighDisagreement).await.unwrap();

    // Start monitoring
    let mut automation = automation;
    automation.start_monitoring().await.unwrap();

    // Wait for action execution
    tokio::time::sleep(Duration::from_millis(500)).await;

    // Verify position reduction was attempted
    let state = automation.get_state().await;
    assert!(state.executed_actions.iter().any(|(a, _)| matches!(a, RollbackAction::ReducePositions)));

    automation.stop_monitoring().await;
}

#[tokio::test]
async fn test_model_disabling_confirmation() {
    let config = RollbackConfig::default();
    let automation = RollbackAutomation::new(config);

    // Trigger model failure scenario
    automation.trigger_scenario_manual(RollbackScenario::ModelFailure).await.unwrap();

    // Start monitoring
    let mut automation = automation;
    automation.start_monitoring().await.unwrap();

    // Wait for action execution
    tokio::time::sleep(Duration::from_millis(500)).await;

    // Verify model disabling was confirmed
    let state = automation.get_state().await;
    assert!(state.executed_actions.iter().any(|(a, _)| matches!(a, RollbackAction::DisableModels)));

    automation.stop_monitoring().await;
}

#[tokio::test]
async fn test_baseline_revert_execution() {
    let config = RollbackConfig::default();
    let automation = RollbackAutomation::new(config);

    // Trigger cascade failure scenario
    automation.trigger_scenario_manual(RollbackScenario::CascadeFailure).await.unwrap();

    // Start monitoring
    let mut automation = automation;
    automation.start_monitoring().await.unwrap();

    // Wait for action execution
    tokio::time::sleep(Duration::from_millis(500)).await;

    // Verify baseline revert was attempted
    let state = automation.get_state().await;
    assert!(state.executed_actions.iter().any(|(a, _)| matches!(a, RollbackAction::RevertToBaseline)));

    automation.stop_monitoring().await;
}

#[tokio::test]
async fn test_daily_loss_scenario_full_recovery() {
    let config = RollbackConfig {
        daily_loss_threshold_usd: 2000.0,
        enable_automatic_rollback: true,
        ..Default::default()
    };
    let automation = RollbackAutomation::new(config);

    // Simulate large daily loss
    automation.update_daily_pnl(-2500.0).await.unwrap();

    // Start monitoring
    let mut automation = automation;
    automation.start_monitoring().await.unwrap();

    // Wait for recovery
    tokio::time::sleep(Duration::from_secs(2)).await;

    // Verify complete recovery
    let state = automation.get_state().await;
    
    // Should trigger DailyLossExceeded scenario
    assert!(state.active_scenarios.contains_key(&RollbackScenario::DailyLossExceeded));
    
    // Should execute EmergencyHalt and ReducePositions
    assert!(state.executed_actions.iter().any(|(a, _)| matches!(a, RollbackAction::EmergencyHalt)));
    assert!(state.executed_actions.iter().any(|(a, _)| matches!(a, RollbackAction::ReducePositions)));
    
    // Trading should be halted
    assert!(!automation.is_trading_enabled());
    
    // Recovery should have started
    assert!(state.recovery_start.is_some());

    automation.stop_monitoring().await;
}

#[tokio::test]
async fn test_high_disagreement_scenario_full_recovery() {
    let config = RollbackConfig {
        high_disagreement_threshold: 0.70,
        disagreement_duration_secs: 10, // 10 seconds for testing
        monitoring_interval_secs: 1,
        enable_automatic_rollback: true,
        ..Default::default()
    };
    let automation = RollbackAutomation::new(config);

    // Record sustained high disagreement
    for _ in 0..15 {
        automation.record_disagreement(0.75).await.unwrap();
        tokio::time::sleep(Duration::from_millis(100)).await;
    }

    // Start monitoring
    let mut automation = automation;
    automation.start_monitoring().await.unwrap();

    // Wait for recovery
    tokio::time::sleep(Duration::from_secs(2)).await;

    // Verify recovery actions
    let state = automation.get_state().await;
    
    // Should execute RevertToBaseline and ReducePositions
    assert!(state.executed_actions.iter().any(|(a, _)| matches!(a, RollbackAction::RevertToBaseline)));
    assert!(state.executed_actions.iter().any(|(a, _)| matches!(a, RollbackAction::ReducePositions)));

    automation.stop_monitoring().await;
}

#[tokio::test]
async fn test_cascade_failure_scenario_full_recovery() {
    let config = RollbackConfig {
        cascade_failure_threshold: 2,
        enable_automatic_rollback: true,
        ..Default::default()
    };
    let automation = RollbackAutomation::new(config);

    // Trigger cascade failure
    automation.trigger_scenario_manual(RollbackScenario::CascadeFailure).await.unwrap();

    // Start monitoring
    let mut automation = automation;
    automation.start_monitoring().await.unwrap();

    // Wait for recovery
    tokio::time::sleep(Duration::from_secs(2)).await;

    // Verify emergency response
    let state = automation.get_state().await;
    
    // Should execute EmergencyHalt and RevertToBaseline
    assert!(state.executed_actions.iter().any(|(a, _)| matches!(a, RollbackAction::EmergencyHalt)));
    assert!(state.executed_actions.iter().any(|(a, _)| matches!(a, RollbackAction::RevertToBaseline)));
    
    // Trading should be halted
    assert!(!automation.is_trading_enabled());

    automation.stop_monitoring().await;
}

#[tokio::test]
async fn test_recovery_duration_tracking() {
    let config = RollbackConfig::default();
    let automation = RollbackAutomation::new(config);

    // Trigger scenario
    automation.trigger_scenario_manual(RollbackScenario::DailyLossExceeded).await.unwrap();

    // Start monitoring
    let mut automation = automation;
    automation.start_monitoring().await.unwrap();

    // Wait for recovery
    tokio::time::sleep(Duration::from_millis(500)).await;

    // Verify recovery duration is tracked
    let duration = automation.get_recovery_duration().await;
    assert!(duration.is_some());
    assert!(duration.unwrap().as_millis() >= 100);
    assert!(duration.unwrap() < Duration::from_secs(300)); // <5 minutes

    automation.stop_monitoring().await;
}

#[tokio::test]
async fn test_rollback_report_generation() {
    let config = RollbackConfig::default();
    let automation = RollbackAutomation::new(config);

    // Trigger multiple scenarios
    automation.trigger_scenario_manual(RollbackScenario::DailyLossExceeded).await.unwrap();
    automation.trigger_scenario_manual(RollbackScenario::ModelFailure).await.unwrap();

    // Start monitoring
    let mut automation = automation;
    automation.start_monitoring().await.unwrap();

    // Wait for recovery
    tokio::time::sleep(Duration::from_secs(1)).await;

    // Generate report
    let state = automation.get_state().await;
    let report = trading_service::rollback_automation::RollbackReport::from_state(&state);

    // Verify report content
    assert!(!report.scenarios_triggered.is_empty());
    assert!(!report.actions_executed.is_empty());
    assert!(report.recovery_duration.is_some());
    assert!(report.trading_halted);

    automation.stop_monitoring().await;
}

#[tokio::test]
async fn test_automatic_vs_manual_rollback() {
    // Test with automatic rollback disabled
    let config_manual = RollbackConfig {
        enable_automatic_rollback: false,
        ..Default::default()
    };
    let automation_manual = RollbackAutomation::new(config_manual);

    automation_manual.trigger_scenario_manual(RollbackScenario::DailyLossExceeded).await.unwrap();
    
    let mut automation_manual = automation_manual;
    automation_manual.start_monitoring().await.unwrap();
    tokio::time::sleep(Duration::from_millis(500)).await;

    // Should detect but not execute
    let state_manual = automation_manual.get_state().await;
    assert!(!state_manual.active_scenarios.is_empty());
    assert!(state_manual.executed_actions.is_empty()); // No actions when disabled

    automation_manual.stop_monitoring().await;

    // Test with automatic rollback enabled
    let config_auto = RollbackConfig {
        enable_automatic_rollback: true,
        ..Default::default()
    };
    let automation_auto = RollbackAutomation::new(config_auto);

    automation_auto.trigger_scenario_manual(RollbackScenario::DailyLossExceeded).await.unwrap();
    
    let mut automation_auto = automation_auto;
    automation_auto.start_monitoring().await.unwrap();
    tokio::time::sleep(Duration::from_millis(500)).await;

    // Should detect AND execute
    let state_auto = automation_auto.get_state().await;
    assert!(!state_auto.active_scenarios.is_empty());
    assert!(!state_auto.executed_actions.is_empty()); // Actions executed

    automation_auto.stop_monitoring().await;
}

#[tokio::test]
async fn test_reset_functionality() {
    let config = RollbackConfig::default();
    let automation = RollbackAutomation::new(config);

    // Trigger scenarios and execute actions
    automation.update_daily_pnl(-3000.0).await.unwrap();
    automation.trigger_scenario_manual(RollbackScenario::DailyLossExceeded).await.unwrap();

    let mut automation = automation;
    automation.start_monitoring().await.unwrap();
    tokio::time::sleep(Duration::from_millis(500)).await;

    // Verify state is dirty
    let state_before = automation.get_state().await;
    assert!(!state_before.active_scenarios.is_empty());
    assert!(!state_before.executed_actions.is_empty());

    // Reset
    automation.reset_all().await.unwrap();

    // Verify state is clean
    let state_after = automation.get_state().await;
    assert_eq!(state_after.daily_pnl_usd, 0.0);
    assert!(state_after.active_scenarios.is_empty());
    assert!(state_after.executed_actions.is_empty());
    assert!(!state_after.trading_halted);

    automation.stop_monitoring().await;
}