foxhunt/services/trading_service/tests/rollback_automation_tests.rs

//! Comprehensive Integration Tests for Rollback Automation
//!
//! Tests all 4 failure scenarios with automatic recovery:
//! 1. Daily loss exceeds $2K
//! 2. Model disagreement >70% for 1 hour
//! 3. Single model >3 consecutive errors
//! 4. Cascade failure (2+ models fail)

use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use tokio::time::sleep;

use trading_service::ensemble_coordinator::EnsembleCoordinator;
use trading_service::ensemble_risk_manager::{EnsembleRiskConfig, EnsembleRiskManager};
use trading_service::rollback_automation::{
    RollbackAutomation, RollbackConfig, RollbackReport, RollbackScenario, RollbackAction,
};

/// Test helper to create automation with default config
fn create_automation() -> RollbackAutomation {
    let config = RollbackConfig {
        daily_loss_threshold_usd: 2000.0,
        high_disagreement_threshold: 0.70,
        disagreement_duration_secs: 10, // 10 seconds for testing
        max_consecutive_errors: 3,
        cascade_failure_threshold: 2,
        position_reduction_factor: 0.50,
        monitoring_interval_secs: 1, // 1 second for testing
        recovery_timeout_secs: 300,
        enable_automatic_rollback: true,
    };

    RollbackAutomation::new(config)
}

/// Test helper to create ensemble risk manager
async fn create_risk_manager() -> Arc<EnsembleRiskManager> {
    let config = EnsembleRiskConfig {
        max_consecutive_errors: 3,
        cascade_failure_threshold: 2,
        cascade_detection_window_secs: 60,
        ..Default::default()
    };

    let manager = EnsembleRiskManager::new(config);

    // Register test models
    manager.register_model("DQN".to_string()).await.unwrap();
    manager.register_model("PPO".to_string()).await.unwrap();
    manager.register_model("TFT".to_string()).await.unwrap();

    Arc::new(manager)
}

/// Test helper to create ensemble coordinator
async fn create_coordinator() -> Arc<EnsembleCoordinator> {
    let coordinator = EnsembleCoordinator::new();

    coordinator.register_model("DQN".to_string(), 0.33).await.unwrap();
    coordinator.register_model("PPO".to_string(), 0.33).await.unwrap();
    coordinator.register_model("TFT".to_string(), 0.34).await.unwrap();

    Arc::new(coordinator)
}

// ============================================================================
// SCENARIO 1: Daily Loss Exceeds $2K
// ============================================================================

#[tokio::test]
async fn test_scenario_1_daily_loss_exceeded_basic() {
    let automation = create_automation();

    // Simulate daily loss exceeding threshold
    automation.update_daily_pnl(-2500.0).await.unwrap();

    // Trigger scenario manually
    automation.trigger_scenario_manual(RollbackScenario::DailyLossExceeded).await.unwrap();

    let state = automation.get_state().await;
    assert!(state.active_scenarios.contains_key(&RollbackScenario::DailyLossExceeded));
    assert_eq!(state.daily_pnl_usd, -2500.0);
}

#[tokio::test]
async fn test_scenario_1_emergency_halt_executed() {
    let automation = create_automation();

    // Trigger scenario
    automation.trigger_scenario_manual(RollbackScenario::DailyLossExceeded).await.unwrap();

    // Execute recovery
    let config = automation.config.clone();
    RollbackAutomation::execute_recovery_actions(&config, &automation.state).await.unwrap();

    // Verify emergency halt
    assert!(automation.is_trading_halted().await);

    let state = automation.get_state().await;
    assert!(state.executed_actions.iter().any(|(a, _)| *a == RollbackAction::EmergencyHalt));
}

#[tokio::test]
async fn test_scenario_1_position_reduction_executed() {
    let automation = create_automation();

    // Trigger scenario
    automation.trigger_scenario_manual(RollbackScenario::DailyLossExceeded).await.unwrap();

    // Execute recovery
    let config = automation.config.clone();
    RollbackAutomation::execute_recovery_actions(&config, &automation.state).await.unwrap();

    // Verify position reduction
    assert!(automation.are_positions_reduced().await);

    let state = automation.get_state().await;
    assert!(state.executed_actions.iter().any(|(a, _)| *a == RollbackAction::ReducePositions));
}

#[tokio::test]
async fn test_scenario_1_recovery_time_under_5_minutes() {
    let automation = create_automation();

    // Trigger and recover
    automation.trigger_scenario_manual(RollbackScenario::DailyLossExceeded).await.unwrap();

    let config = automation.config.clone();
    RollbackAutomation::execute_recovery_actions(&config, &automation.state).await.unwrap();

    // Check recovery time
    let duration = automation.get_recovery_duration().await;
    assert!(duration.is_some());
    assert!(duration.unwrap().as_secs() < 300); // Less than 5 minutes
}

#[tokio::test]
async fn test_scenario_1_full_recovery_sequence() {
    let automation = create_automation();

    // Simulate loss
    automation.update_daily_pnl(-2500.0).await.unwrap();

    // Trigger
    automation.trigger_scenario_manual(RollbackScenario::DailyLossExceeded).await.unwrap();

    // Execute recovery
    let config = automation.config.clone();
    RollbackAutomation::execute_recovery_actions(&config, &automation.state).await.unwrap();

    // Verify complete recovery
    let state = automation.get_state().await;
    let report = RollbackReport::from_state(&state);

    assert!(report.trading_halted);
    assert!(report.positions_reduced);
    assert!(!report.actions_executed.is_empty());
}

// ============================================================================
// SCENARIO 2: Model Disagreement >70% for 1 Hour
// ============================================================================

#[tokio::test]
async fn test_scenario_2_high_disagreement_detection() {
    let automation = create_automation();

    // Record sustained high disagreement
    for _ in 0..15 {
        automation.record_disagreement(0.75).await.unwrap();
        sleep(Duration::from_millis(100)).await;
    }

    // Trigger scenario
    automation.trigger_scenario_manual(RollbackScenario::HighDisagreement).await.unwrap();

    let state = automation.get_state().await;
    assert!(state.active_scenarios.contains_key(&RollbackScenario::HighDisagreement));
}

#[tokio::test]
async fn test_scenario_2_baseline_revert_executed() {
    let automation = create_automation();

    // Trigger scenario
    automation.trigger_scenario_manual(RollbackScenario::HighDisagreement).await.unwrap();

    // Execute recovery
    let config = automation.config.clone();
    RollbackAutomation::execute_recovery_actions(&config, &automation.state).await.unwrap();

    // Verify baseline mode
    assert!(automation.is_baseline_mode_active().await);

    let state = automation.get_state().await;
    assert!(state.executed_actions.iter().any(|(a, _)| *a == RollbackAction::RevertToBaseline));
}

#[tokio::test]
async fn test_scenario_2_position_reduction() {
    let automation = create_automation();

    // Trigger scenario
    automation.trigger_scenario_manual(RollbackScenario::HighDisagreement).await.unwrap();

    // Execute recovery
    let config = automation.config.clone();
    RollbackAutomation::execute_recovery_actions(&config, &automation.state).await.unwrap();

    // Verify position reduction
    assert!(automation.are_positions_reduced().await);
}

#[tokio::test]
async fn test_scenario_2_disagreement_windowing() {
    let automation = create_automation();

    // Record disagreement within window
    for _ in 0..10 {
        automation.record_disagreement(0.75).await.unwrap();
    }

    let state = automation.get_state().await;
    assert!(!state.disagreement_history.is_empty());
    assert!(state.disagreement_history.len() <= 10);
}

#[tokio::test]
async fn test_scenario_2_recovery_time() {
    let automation = create_automation();

    // Trigger and recover
    automation.trigger_scenario_manual(RollbackScenario::HighDisagreement).await.unwrap();

    let config = automation.config.clone();
    RollbackAutomation::execute_recovery_actions(&config, &automation.state).await.unwrap();

    // Check recovery time
    let duration = automation.get_recovery_duration().await;
    assert!(duration.is_some());
    assert!(duration.unwrap().as_secs() < 300);
}

// ============================================================================
// SCENARIO 3: Single Model >3 Consecutive Errors
// ============================================================================

#[tokio::test]
async fn test_scenario_3_model_failure_detection() {
    let risk_manager = create_risk_manager().await;

    // Simulate 3 consecutive errors for DQN
    for _ in 0..3 {
        risk_manager.record_prediction_result("DQN", false).await.unwrap();
    }

    // Check model health
    let health = risk_manager.get_model_health("DQN").await.unwrap();
    assert!(!health.enabled);
    assert_eq!(health.consecutive_errors, 3);
}

#[tokio::test]
async fn test_scenario_3_model_disabled() {
    let risk_manager = create_risk_manager().await;
    let automation = create_automation()
        .with_ensemble_risk_manager(Arc::clone(&risk_manager));

    // Fail DQN model
    for _ in 0..3 {
        risk_manager.record_prediction_result("DQN", false).await.unwrap();
    }

    // Trigger scenario
    automation.trigger_scenario_manual(RollbackScenario::ModelFailure).await.unwrap();

    // Execute recovery
    let config = automation.config.clone();
    RollbackAutomation::execute_recovery_actions(&config, &automation.state).await.unwrap();

    let state = automation.get_state().await;
    assert!(state.executed_actions.iter().any(|(a, _)| *a == RollbackAction::DisableModels));
}

#[tokio::test]
async fn test_scenario_3_baseline_mode_activated() {
    let risk_manager = create_risk_manager().await;
    let automation = create_automation()
        .with_ensemble_risk_manager(Arc::clone(&risk_manager));

    // Trigger model failure
    automation.trigger_scenario_manual(RollbackScenario::ModelFailure).await.unwrap();

    // Execute recovery
    let config = automation.config.clone();
    RollbackAutomation::execute_recovery_actions(&config, &automation.state).await.unwrap();

    // Verify baseline mode
    assert!(automation.is_baseline_mode_active().await);
}

#[tokio::test]
async fn test_scenario_3_successful_prediction_resets_errors() {
    let risk_manager = create_risk_manager().await;

    // Record 2 errors
    risk_manager.record_prediction_result("DQN", false).await.unwrap();
    risk_manager.record_prediction_result("DQN", false).await.unwrap();

    // Record success (should reset counter)
    risk_manager.record_prediction_result("DQN", true).await.unwrap();

    let health = risk_manager.get_model_health("DQN").await.unwrap();
    assert!(health.enabled);
    assert_eq!(health.consecutive_errors, 0);
}

#[tokio::test]
async fn test_scenario_3_recovery_time() {
    let risk_manager = create_risk_manager().await;
    let automation = create_automation()
        .with_ensemble_risk_manager(Arc::clone(&risk_manager));

    // Trigger and recover
    automation.trigger_scenario_manual(RollbackScenario::ModelFailure).await.unwrap();

    let config = automation.config.clone();
    RollbackAutomation::execute_recovery_actions(&config, &automation.state).await.unwrap();

    let duration = automation.get_recovery_duration().await;
    assert!(duration.is_some());
    assert!(duration.unwrap().as_secs() < 300);
}

// ============================================================================
// SCENARIO 4: Cascade Failure (2+ Models Fail)
// ============================================================================

#[tokio::test]
async fn test_scenario_4_cascade_failure_detection() {
    let risk_manager = create_risk_manager().await;

    // Fail DQN
    for _ in 0..3 {
        risk_manager.record_prediction_result("DQN", false).await.unwrap();
    }

    // Fail PPO
    for _ in 0..3 {
        risk_manager.record_prediction_result("PPO", false).await.unwrap();
    }

    // Check cascade state
    let cascade_state = risk_manager.get_cascade_state().await;
    assert!(cascade_state.is_cascading);
    assert_eq!(cascade_state.failed_models.len(), 2);
}

#[tokio::test]
async fn test_scenario_4_emergency_halt() {
    let risk_manager = create_risk_manager().await;
    let automation = create_automation()
        .with_ensemble_risk_manager(Arc::clone(&risk_manager));

    // Trigger cascade failure
    automation.trigger_scenario_manual(RollbackScenario::CascadeFailure).await.unwrap();

    // Execute recovery
    let config = automation.config.clone();
    RollbackAutomation::execute_recovery_actions(&config, &automation.state).await.unwrap();

    // Verify emergency halt
    assert!(automation.is_trading_halted().await);
}

#[tokio::test]
async fn test_scenario_4_baseline_revert() {
    let risk_manager = create_risk_manager().await;
    let automation = create_automation()
        .with_ensemble_risk_manager(Arc::clone(&risk_manager));

    // Trigger cascade failure
    automation.trigger_scenario_manual(RollbackScenario::CascadeFailure).await.unwrap();

    // Execute recovery
    let config = automation.config.clone();
    RollbackAutomation::execute_recovery_actions(&config, &automation.state).await.unwrap();

    // Verify baseline mode
    assert!(automation.is_baseline_mode_active().await);
}

#[tokio::test]
async fn test_scenario_4_cascade_within_window() {
    let risk_manager = create_risk_manager().await;

    // Fail models within detection window
    for _ in 0..3 {
        risk_manager.record_prediction_result("DQN", false).await.unwrap();
    }

    sleep(Duration::from_millis(100)).await;

    for _ in 0..3 {
        risk_manager.record_prediction_result("PPO", false).await.unwrap();
    }

    let cascade_state = risk_manager.get_cascade_state().await;
    assert!(cascade_state.is_cascading);
}

#[tokio::test]
async fn test_scenario_4_recovery_time() {
    let risk_manager = create_risk_manager().await;
    let automation = create_automation()
        .with_ensemble_risk_manager(Arc::clone(&risk_manager));

    // Trigger and recover
    automation.trigger_scenario_manual(RollbackScenario::CascadeFailure).await.unwrap();

    let config = automation.config.clone();
    RollbackAutomation::execute_recovery_actions(&config, &automation.state).await.unwrap();

    let duration = automation.get_recovery_duration().await;
    assert!(duration.is_some());
    assert!(duration.unwrap().as_secs() < 300);
}

// ============================================================================
// COMPREHENSIVE INTEGRATION TESTS
// ============================================================================

#[tokio::test]
async fn test_all_scenarios_sequential() {
    let risk_manager = create_risk_manager().await;
    let automation = create_automation()
        .with_ensemble_risk_manager(Arc::clone(&risk_manager));

    // Scenario 1: Daily loss
    automation.update_daily_pnl(-2500.0).await.unwrap();
    automation.trigger_scenario_manual(RollbackScenario::DailyLossExceeded).await.unwrap();

    // Scenario 2: High disagreement
    for _ in 0..10 {
        automation.record_disagreement(0.75).await.unwrap();
    }
    automation.trigger_scenario_manual(RollbackScenario::HighDisagreement).await.unwrap();

    // Scenario 3: Model failure
    automation.trigger_scenario_manual(RollbackScenario::ModelFailure).await.unwrap();

    // Scenario 4: Cascade failure
    automation.trigger_scenario_manual(RollbackScenario::CascadeFailure).await.unwrap();

    // Execute recovery
    let config = automation.config.clone();
    RollbackAutomation::execute_recovery_actions(&config, &automation.state).await.unwrap();

    // Verify all actions executed
    let state = automation.get_state().await;
    assert!(state.trading_halted);
    assert!(state.positions_reduced);
    assert!(state.baseline_mode_active);
    assert_eq!(state.active_scenarios.len(), 4);
}

#[tokio::test]
async fn test_recovery_report_generation() {
    let automation = create_automation();

    // Trigger scenarios
    automation.trigger_scenario_manual(RollbackScenario::DailyLossExceeded).await.unwrap();
    automation.trigger_scenario_manual(RollbackScenario::HighDisagreement).await.unwrap();

    // Execute recovery
    let config = automation.config.clone();
    RollbackAutomation::execute_recovery_actions(&config, &automation.state).await.unwrap();

    // Generate report
    let state = automation.get_state().await;
    let report = RollbackReport::from_state(&state);

    assert_eq!(report.scenarios_triggered.len(), 2);
    assert!(!report.actions_executed.is_empty());
    assert!(report.recovery_duration.is_some());
}

#[tokio::test]
async fn test_success_criteria_validation() {
    let automation = create_automation();

    // Trigger and recover quickly
    automation.trigger_scenario_manual(RollbackScenario::DailyLossExceeded).await.unwrap();

    let config = automation.config.clone();
    RollbackAutomation::execute_recovery_actions(&config, &automation.state).await.unwrap();

    // Wait briefly
    sleep(Duration::from_millis(100)).await;

    // Complete recovery
    let state = automation.get_state().await;
    let report = RollbackReport::from_state(&state);

    // Verify meets success criteria if recovery completed
    if report.recovery_completed {
        assert!(report.meets_success_criteria());
    }
}

#[tokio::test]
async fn test_action_priority_execution() {
    let automation = create_automation();

    // Trigger cascade (should execute EmergencyHalt first)
    automation.trigger_scenario_manual(RollbackScenario::CascadeFailure).await.unwrap();

    let config = automation.config.clone();
    RollbackAutomation::execute_recovery_actions(&config, &automation.state).await.unwrap();

    let state = automation.get_state().await;

    // Find EmergencyHalt action
    let halt_idx = state.executed_actions.iter().position(|(a, _)| *a == RollbackAction::EmergencyHalt);

    // Find other actions
    let baseline_idx = state.executed_actions.iter().position(|(a, _)| *a == RollbackAction::RevertToBaseline);

    // EmergencyHalt should execute before or at same time as baseline
    if let (Some(halt), Some(baseline)) = (halt_idx, baseline_idx) {
        assert!(halt <= baseline);
    }
}

#[tokio::test]
async fn test_idempotent_action_execution() {
    let automation = create_automation();

    // Trigger scenario
    automation.trigger_scenario_manual(RollbackScenario::DailyLossExceeded).await.unwrap();

    // Execute recovery multiple times
    let config = automation.config.clone();
    for _ in 0..3 {
        RollbackAutomation::execute_recovery_actions(&config, &automation.state).await.unwrap();
    }

    let state = automation.get_state().await;

    // Each action should only be executed once
    let halt_count = state.executed_actions.iter().filter(|(a, _)| *a == RollbackAction::EmergencyHalt).count();
    assert_eq!(halt_count, 1);
}

#[tokio::test]
async fn test_reset_functionality() {
    let automation = create_automation();

    // Trigger scenarios and recover
    automation.trigger_scenario_manual(RollbackScenario::DailyLossExceeded).await.unwrap();
    automation.update_daily_pnl(-3000.0).await.unwrap();

    let config = automation.config.clone();
    RollbackAutomation::execute_recovery_actions(&config, &automation.state).await.unwrap();

    // Reset
    automation.reset_all().await.unwrap();

    // Verify clean state
    let state = automation.get_state().await;
    assert_eq!(state.daily_pnl_usd, 0.0);
    assert!(state.active_scenarios.is_empty());
    assert!(state.executed_actions.is_empty());
    assert!(!state.trading_halted);
    assert!(!state.positions_reduced);
    assert!(!state.baseline_mode_active);
}

#[tokio::test]
async fn test_disabled_automatic_rollback() {
    let config = RollbackConfig {
        enable_automatic_rollback: false,
        ..Default::default()
    };
    let automation = RollbackAutomation::new(config.clone());

    // Trigger scenario
    automation.trigger_scenario_manual(RollbackScenario::DailyLossExceeded).await.unwrap();

    // Execute recovery (should do nothing)
    RollbackAutomation::execute_recovery_actions(&config, &automation.state).await.unwrap();

    let state = automation.get_state().await;
    assert!(state.executed_actions.is_empty());
    assert!(!state.trading_halted);
}

#[tokio::test]
async fn test_daily_reset() {
    let automation = create_automation();

    // Set P&L
    automation.update_daily_pnl(-1500.0).await.unwrap();

    // Reset daily
    automation.reset_daily().await.unwrap();

    let state = automation.get_state().await;
    assert_eq!(state.daily_pnl_usd, 0.0);
}

#[tokio::test]
async fn test_recovery_timeout_detection() {
    let config = RollbackConfig {
        recovery_timeout_secs: 1, // 1 second timeout
        ..Default::default()
    };
    let automation = RollbackAutomation::new(config.clone());

    // Trigger scenario (starts recovery)
    automation.trigger_scenario_manual(RollbackScenario::DailyLossExceeded).await.unwrap();
    RollbackAutomation::execute_recovery_actions(&config, &automation.state).await.unwrap();

    // Wait for timeout
    sleep(Duration::from_secs(2)).await;

    // Check timeout (would be logged as error)
    let duration = automation.get_recovery_duration().await;
    assert!(duration.is_some());
    assert!(duration.unwrap().as_secs() >= 1);
}

// ============================================================================
// STRESS TESTS
// ============================================================================

#[tokio::test]
async fn test_rapid_scenario_triggers() {
    let automation = create_automation();

    // Rapidly trigger scenarios
    for _ in 0..10 {
        automation.trigger_scenario_manual(RollbackScenario::DailyLossExceeded).await.unwrap();
        sleep(Duration::from_millis(10)).await;
    }

    let state = automation.get_state().await;
    assert!(state.active_scenarios.contains_key(&RollbackScenario::DailyLossExceeded));
}

#[tokio::test]
async fn test_concurrent_disagreement_recording() {
    let automation = Arc::new(create_automation());

    // Spawn concurrent tasks recording disagreement
    let mut handles = vec![];
    for i in 0..10 {
        let auto = Arc::clone(&automation);
        let handle = tokio::spawn(async move {
            auto.record_disagreement(0.75 + (i as f64 * 0.01)).await.unwrap();
        });
        handles.push(handle);
    }

    // Wait for all tasks
    for handle in handles {
        handle.await.unwrap();
    }

    let state = automation.get_state().await;
    assert!(!state.disagreement_history.is_empty());
}

#[tokio::test]
async fn test_high_frequency_pnl_updates() {
    let automation = create_automation();

    // Rapidly update P&L
    for i in 0..100 {
        let pnl = -1000.0 - (i as f64 * 10.0);
        automation.update_daily_pnl(pnl).await.unwrap();
    }

    let state = automation.get_state().await;
    assert!(state.daily_pnl_usd < -2000.0);
}