foxhunt/services/ml_training_service/tests/deployment_tests.rs

//! TDD Tests for Automated Model Deployment Pipeline
//!
//! **TDD Approach**: Tests written FIRST, implementation follows to make them GREEN
//!
//! Test Coverage:
//! 1. Deployment trigger on A/B test pass
//! 2. Rolling update with zero downtime
//! 3. Health check validation (model inference working)
//! 4. Rollback on health check failure
//! 5. E2E deployment with real model

use anyhow::Result;
use ml_training_service::deployment_pipeline::{
    ABTestResult, DeploymentConfig, DeploymentPipeline, DeploymentResult, DeploymentStatus,
    GroupMetrics, HealthCheckConfig,
    RollbackStrategy, RollingUpdateConfig,
};
use uuid::Uuid;

// ==================== TEST 1: Deployment Trigger on A/B Test Pass ====================

#[tokio::test]
async fn test_deployment_triggers_on_ab_test_pass() {
    // Arrange
    let config = DeploymentConfig {
        enable_auto_deployment: true,
        trigger_on_ab_test_pass: true,
        min_ab_test_confidence: 0.95,
        ..Default::default()
    };

    let pipeline = DeploymentPipeline::new(config).unwrap();
    let model_id = Uuid::new_v4();

    // Simulate A/B test result that passes thresholds
    let ab_test_result = create_passing_ab_test_result(model_id);

    // Act
    let deployment_result = pipeline
        .trigger_deployment_on_ab_test(ab_test_result)
        .await;

    // Assert
    assert!(deployment_result.is_ok());
    let result = deployment_result.unwrap();
    assert_eq!(result.status, DeploymentStatus::Triggered);
    assert_eq!(result.model_id, model_id);
    assert!(result.triggered_by_ab_test);
}

#[tokio::test]
async fn test_deployment_skips_on_ab_test_fail() {
    // Arrange
    let config = DeploymentConfig {
        enable_auto_deployment: true,
        trigger_on_ab_test_pass: true,
        min_ab_test_confidence: 0.95,
        ..Default::default()
    };

    let pipeline = DeploymentPipeline::new(config).unwrap();
    let model_id = Uuid::new_v4();

    // Simulate A/B test result that fails thresholds
    let ab_test_result = create_failing_ab_test_result(model_id);

    // Act
    let deployment_result = pipeline
        .trigger_deployment_on_ab_test(ab_test_result)
        .await;

    // Assert
    assert!(deployment_result.is_ok());
    let result = deployment_result.unwrap();
    assert_eq!(result.status, DeploymentStatus::Skipped);
    assert!(!result.triggered_by_ab_test);
}

// ==================== TEST 2: Rolling Update (Zero Downtime) ====================

#[tokio::test]
async fn test_rolling_update_zero_downtime() {
    // Arrange
    let config = DeploymentConfig {
        enable_auto_deployment: true,
        rolling_update: RollingUpdateConfig {
            batch_size: 1,
            batch_delay_seconds: 1,
            health_check_retries: 3,
            health_check_interval_seconds: 1,
        },
        min_ab_test_confidence: 0.95,
        ..Default::default()
    };

    let pipeline = DeploymentPipeline::new(config).unwrap();
    let model_id = Uuid::new_v4();
    let model_path = format!("/tmp/models/{}/model.safetensors", model_id);

    // Act
    let deployment_result = pipeline
        .perform_rolling_update(model_id, &model_path, 3) // 3 instances
        .await;

    // Assert
    assert!(deployment_result.is_ok());
    let result = deployment_result.unwrap();
    assert_eq!(result.status, DeploymentStatus::Completed);
    assert_eq!(result.instances_updated, 3);
    assert!(result.zero_downtime_achieved);
    assert!(result.deployment_duration_seconds < 10); // Should complete quickly
}

#[tokio::test]
async fn test_rolling_update_respects_batch_size() {
    // Arrange
    let config = DeploymentConfig {
        enable_auto_deployment: true,
        rolling_update: RollingUpdateConfig {
            batch_size: 2, // Update 2 instances at a time
            batch_delay_seconds: 1,
            health_check_retries: 3,
            health_check_interval_seconds: 1,
        },
        min_ab_test_confidence: 0.95,
        ..Default::default()
    };

    let pipeline = DeploymentPipeline::new(config).unwrap();
    let model_id = Uuid::new_v4();
    let model_path = format!("/tmp/models/{}/model.safetensors", model_id);

    // Act
    let start = std::time::Instant::now();
    let deployment_result = pipeline
        .perform_rolling_update(model_id, &model_path, 4) // 4 instances
        .await;
    let elapsed = start.elapsed();

    // Assert
    assert!(deployment_result.is_ok());
    let result = deployment_result.unwrap();
    assert_eq!(result.status, DeploymentStatus::Completed);
    assert_eq!(result.instances_updated, 4);
    assert_eq!(result.batches_executed, 2); // 4 instances / batch_size=2 = 2 batches
    assert!(elapsed.as_secs() >= 1); // Should have delay between batches
}

// ==================== TEST 3: Health Check Validation ====================

#[tokio::test]
async fn test_health_check_validates_model_inference() {
    // Arrange
    let config = DeploymentConfig {
        enable_auto_deployment: true,
        health_check: HealthCheckConfig {
            enabled: true,
            timeout_seconds: 5,
            max_latency_ms: 100,
            test_predictions: 10,
            min_success_rate: 0.95,
        },
        ..Default::default()
    };

    let pipeline = DeploymentPipeline::new(config).unwrap();
    let model_id = Uuid::new_v4();
    let instance_id = "trading-service-1".to_string();

    // Act
    let health_result = pipeline
        .run_health_check(model_id, &instance_id)
        .await;

    // Assert
    assert!(health_result.is_ok());
    let result = health_result.unwrap();
    assert!(result.healthy);
    assert!(result.inference_working);
    assert!(result.latency_ms < 100.0);
    assert!(result.success_rate >= 0.95);
}

#[tokio::test]
async fn test_health_check_fails_on_inference_error() {
    // Arrange
    let config = DeploymentConfig {
        enable_auto_deployment: true,
        health_check: HealthCheckConfig {
            enabled: true,
            timeout_seconds: 5,
            max_latency_ms: 100,
            test_predictions: 10,
            min_success_rate: 0.95,
        },
        ..Default::default()
    };

    let pipeline = DeploymentPipeline::new(config).unwrap();
    let model_id = Uuid::new_v4();
    let instance_id = "trading-service-broken".to_string(); // Simulate broken instance

    // Act
    let health_result = pipeline
        .run_health_check(model_id, &instance_id)
        .await;

    // Assert
    assert!(health_result.is_ok()); // Health check runs, but reports unhealthy
    let result = health_result.unwrap();
    assert!(!result.healthy);
    assert!(!result.inference_working);
}

#[tokio::test]
async fn test_health_check_fails_on_high_latency() {
    // Arrange
    let config = DeploymentConfig {
        enable_auto_deployment: true,
        health_check: HealthCheckConfig {
            enabled: true,
            timeout_seconds: 5,
            max_latency_ms: 10, // Very strict latency requirement
            test_predictions: 10,
            min_success_rate: 0.95,
        },
        ..Default::default()
    };

    let pipeline = DeploymentPipeline::new(config).unwrap();
    let model_id = Uuid::new_v4();
    let instance_id = "trading-service-slow".to_string(); // Simulate slow instance

    // Act
    let health_result = pipeline
        .run_health_check(model_id, &instance_id)
        .await;

    // Assert
    assert!(health_result.is_ok());
    let result = health_result.unwrap();
    assert!(!result.healthy);
    assert!(result.latency_ms > 10.0);
}

// ==================== TEST 4: Rollback on Failure ====================

#[tokio::test]
async fn test_rollback_on_health_check_failure() {
    // Arrange
    let config = DeploymentConfig {
        enable_auto_deployment: true,
        rollback_strategy: RollbackStrategy::Automatic,
        rollback_on_health_check_failure: true,
        ..Default::default()
    };

    let pipeline = DeploymentPipeline::new(config).unwrap();
    let model_id = Uuid::new_v4();
    let previous_model_id = Uuid::new_v4();
    let model_path = format!("/tmp/models/{}/model_broken.safetensors", model_id);

    // Simulate deployment that fails health check
    let deployment_result = pipeline
        .deploy_with_rollback(model_id, previous_model_id, &model_path, 2)
        .await;

    // Assert
    assert!(deployment_result.is_ok());
    let result = deployment_result.unwrap();
    assert_eq!(result.status, DeploymentStatus::RolledBack);
    assert!(result.rollback_triggered);
    assert_eq!(result.active_model_id, previous_model_id); // Should revert to previous
}

#[tokio::test]
async fn test_rollback_restores_previous_model() {
    // Arrange
    let config = DeploymentConfig {
        enable_auto_deployment: true,
        rollback_strategy: RollbackStrategy::Automatic,
        rollback_on_health_check_failure: true,
        ..Default::default()
    };

    let pipeline = DeploymentPipeline::new(config).unwrap();
    let new_model_id = Uuid::new_v4();
    let previous_model_id = Uuid::new_v4();

    // Act - Trigger rollback
    let rollback_result = pipeline
        .rollback_deployment(new_model_id, previous_model_id)
        .await;

    // Assert
    assert!(rollback_result.is_ok());
    let result = rollback_result.unwrap();
    assert!(result.rollback_successful);
    assert_eq!(result.active_model_id, previous_model_id);
    assert!(result.rollback_duration_seconds < 30); // Should be fast
}

#[tokio::test]
async fn test_manual_rollback_strategy() {
    // Arrange
    let config = DeploymentConfig {
        enable_auto_deployment: true,
        rollback_strategy: RollbackStrategy::Manual,
        rollback_on_health_check_failure: false,
        ..Default::default()
    };

    let pipeline = DeploymentPipeline::new(config).unwrap();
    let model_id = Uuid::new_v4();
    let previous_model_id = Uuid::new_v4();
    let model_path = format!("/tmp/models/{}/model_broken.safetensors", model_id);

    // Act - Deploy with broken model
    let deployment_result = pipeline
        .deploy_with_rollback(model_id, previous_model_id, &model_path, 2)
        .await;

    // Assert - Should NOT automatically rollback with Manual strategy
    assert!(deployment_result.is_ok());
    let result = deployment_result.unwrap();
    assert_eq!(result.status, DeploymentStatus::Failed);
    assert!(!result.rollback_triggered); // Manual strategy = no auto-rollback
}

// ==================== TEST 5: E2E Deployment with Real Model ====================

#[tokio::test]
#[ignore] // Run separately: cargo test test_e2e_deployment -- --ignored
async fn test_e2e_deployment_with_real_model() {
    // Arrange
    let config = DeploymentConfig {
        enable_auto_deployment: true,
        trigger_on_ab_test_pass: true,
        rolling_update: RollingUpdateConfig {
            batch_size: 1,
            batch_delay_seconds: 2,
            health_check_retries: 3,
            health_check_interval_seconds: 1,
        },
        health_check: HealthCheckConfig {
            enabled: true,
            timeout_seconds: 10,
            max_latency_ms: 100,
            test_predictions: 20,
            min_success_rate: 0.95,
        },
        rollback_strategy: RollbackStrategy::Automatic,
        rollback_on_health_check_failure: true,
        min_ab_test_confidence: 0.95,
    };

    let pipeline = DeploymentPipeline::new(config).unwrap();

    // Step 1: Create a real trained model (NO MOCKS)
    let model_id = Uuid::new_v4();
    let model_path = create_real_trained_model(model_id).await.unwrap();

    // Step 2: Run A/B test
    let ab_test_result = create_passing_ab_test_result(model_id);

    // Step 3: Trigger deployment
    let trigger_result = pipeline
        .trigger_deployment_on_ab_test(ab_test_result)
        .await
        .unwrap();
    assert_eq!(trigger_result.status, DeploymentStatus::Triggered);

    // Step 4: Perform rolling update
    let deployment_result = pipeline
        .perform_rolling_update(model_id, &model_path, 3)
        .await
        .unwrap();
    assert_eq!(deployment_result.status, DeploymentStatus::Completed);
    assert_eq!(deployment_result.instances_updated, 3);

    // Step 5: Verify all instances are healthy
    for instance_id in &deployment_result.updated_instances {
        let health = pipeline
            .run_health_check(model_id, instance_id)
            .await
            .unwrap();
        assert!(health.healthy);
        assert!(health.inference_working);
    }
}

// ==================== TEST 6: Deployment Monitoring ====================

#[tokio::test]
async fn test_deployment_status_tracking() {
    // Arrange
    let config = DeploymentConfig::default();
    let pipeline = DeploymentPipeline::new(config).unwrap();
    let model_id = Uuid::new_v4();
    let deployment_id = Uuid::new_v4();

    // Act - Start deployment
    pipeline.start_deployment(deployment_id, model_id).await.unwrap();

    // Query status
    let status = pipeline.get_deployment_status(deployment_id).await.unwrap();

    // Assert
    assert_eq!(status.deployment_id, deployment_id);
    assert_eq!(status.model_id, model_id);
    assert!(matches!(
        status.status,
        DeploymentStatus::InProgress | DeploymentStatus::Triggered
    ));
}

#[tokio::test]
async fn test_deployment_history_tracking() {
    // Arrange
    let config = DeploymentConfig::default();
    let pipeline = DeploymentPipeline::new(config).unwrap();

    // Act - Perform multiple deployments
    for _ in 0..3 {
        let model_id = Uuid::new_v4();
        let model_path = format!("/tmp/models/{}/model.safetensors", model_id);
        let _ = pipeline
            .perform_rolling_update(model_id, &model_path, 1)
            .await;
    }

    // Query history
    let history = pipeline.get_deployment_history(10).await.unwrap();

    // Assert
    assert!(history.len() >= 3);
    for deployment in &history {
        assert!(matches!(
            deployment.status,
            DeploymentStatus::Completed | DeploymentStatus::Failed | DeploymentStatus::RolledBack
        ));
    }
}

// ==================== TEST 7: Concurrent Deployment Prevention ====================

#[tokio::test]
async fn test_prevents_concurrent_deployments() {
    // Arrange
    let config = DeploymentConfig::default();
    let pipeline = DeploymentPipeline::new(config).unwrap();

    // Act - Start first deployment
    let model_id_1 = Uuid::new_v4();
    let model_path_1 = format!("/tmp/models/{}/model.safetensors", model_id_1);
    let deployment_1 = pipeline.perform_rolling_update(model_id_1, &model_path_1, 2);

    // Try to start second deployment concurrently
    let model_id_2 = Uuid::new_v4();
    let model_path_2 = format!("/tmp/models/{}/model.safetensors", model_id_2);
    let deployment_2 = pipeline.perform_rolling_update(model_id_2, &model_path_2, 2);

    // Wait for both
    let (result_1, result_2) = tokio::join!(deployment_1, deployment_2);

    // Assert - One should succeed, other should be rejected
    assert!(result_1.is_ok() || result_2.is_ok()); // At least one succeeds
    assert!(result_1.is_err() || result_2.is_err()); // At least one fails (concurrent rejection)
}

// ==================== HELPER FUNCTIONS ====================

/// Create A/B test result that passes thresholds
fn create_passing_ab_test_result(model_id: Uuid) -> ABTestResult {
    ABTestResult {
        experiment_id: Uuid::new_v4(),
        model_id,
        control_metrics: GroupMetrics {
            avg_latency_ms: 50.0,
            error_rate: 0.01,
            sharpe_ratio: 1.5,
        },
        treatment_metrics: GroupMetrics {
            avg_latency_ms: 45.0, // Better latency
            error_rate: 0.005,   // Lower error rate
            sharpe_ratio: 1.8,   // Higher Sharpe ratio
        },
        statistical_significance: 0.99, // High confidence
        p_value: 0.001,
        passed: true,
    }
}

/// Create A/B test result that fails thresholds
fn create_failing_ab_test_result(model_id: Uuid) -> ABTestResult {
    ABTestResult {
        experiment_id: Uuid::new_v4(),
        model_id,
        control_metrics: GroupMetrics {
            avg_latency_ms: 50.0,
            error_rate: 0.01,
            sharpe_ratio: 1.5,
        },
        treatment_metrics: GroupMetrics {
            avg_latency_ms: 80.0,  // Worse latency
            error_rate: 0.05,     // Higher error rate
            sharpe_ratio: 1.2,    // Lower Sharpe ratio
        },
        statistical_significance: 0.85, // Low confidence
        p_value: 0.15,
        passed: false,
    }
}

/// Create real trained model for E2E test (NO MOCKS)
async fn create_real_trained_model(model_id: Uuid) -> Result<String> {
    use std::path::PathBuf;

    mod test_helpers;

    let model_dir = PathBuf::from(format!("/tmp/models/{}", model_id));
    tokio::fs::create_dir_all(&model_dir).await?;

    // Create real DQN checkpoint using test_helpers
    let checkpoint_path = test_helpers::create_real_dqn_checkpoint(&model_dir, model_id).await?;

    Ok(checkpoint_path.to_string_lossy().to_string())
}