//! TDD Tests for Automated Model Deployment Pipeline //! //! **TDD Approach**: Tests written FIRST, implementation follows to make them GREEN //! //! Test Coverage: //! 1. Deployment trigger on A/B test pass //! 2. Rolling update with zero downtime //! 3. Health check validation (model inference working) //! 4. Rollback on health check failure //! 5. E2E deployment with real model use anyhow::Result; use ml_training_service::deployment_pipeline::{ ABTestResult, DeploymentConfig, DeploymentPipeline, DeploymentStatus, GroupMetrics, HealthCheckConfig, RollbackStrategy, RollingUpdateConfig, }; use uuid::Uuid; // ==================== TEST 1: Deployment Trigger on A/B Test Pass ==================== #[tokio::test] async fn test_deployment_triggers_on_ab_test_pass() { // Arrange let config = DeploymentConfig { enable_auto_deployment: true, trigger_on_ab_test_pass: true, min_ab_test_confidence: 0.95, ..Default::default() }; let pipeline = DeploymentPipeline::new(config).unwrap(); let model_id = Uuid::new_v4(); // Simulate A/B test result that passes thresholds let ab_test_result = create_passing_ab_test_result(model_id); // Act let deployment_result = pipeline.trigger_deployment_on_ab_test(ab_test_result).await; // Assert assert!(deployment_result.is_ok()); let result = deployment_result.unwrap(); assert_eq!(result.status, DeploymentStatus::Triggered); assert_eq!(result.model_id, model_id); assert!(result.triggered_by_ab_test); } #[tokio::test] async fn test_deployment_skips_on_ab_test_fail() { // Arrange let config = DeploymentConfig { enable_auto_deployment: true, trigger_on_ab_test_pass: true, min_ab_test_confidence: 0.95, ..Default::default() }; let pipeline = DeploymentPipeline::new(config).unwrap(); let model_id = Uuid::new_v4(); // Simulate A/B test result that fails thresholds let ab_test_result = create_failing_ab_test_result(model_id); // Act let deployment_result = pipeline.trigger_deployment_on_ab_test(ab_test_result).await; // Assert assert!(deployment_result.is_ok()); let result = deployment_result.unwrap(); assert_eq!(result.status, DeploymentStatus::Skipped); assert!(!result.triggered_by_ab_test); } // ==================== TEST 2: Rolling Update (Zero Downtime) ==================== #[tokio::test] async fn test_rolling_update_zero_downtime() { // Arrange let config = DeploymentConfig { enable_auto_deployment: true, rolling_update: RollingUpdateConfig { batch_size: 1, batch_delay_seconds: 1, health_check_retries: 3, health_check_interval_seconds: 1, }, min_ab_test_confidence: 0.95, ..Default::default() }; let pipeline = DeploymentPipeline::new(config).unwrap(); let model_id = Uuid::new_v4(); let model_path = format!("/tmp/models/{}/model.safetensors", model_id); // Act let deployment_result = pipeline .perform_rolling_update(model_id, &model_path, 3) // 3 instances .await; // Assert assert!(deployment_result.is_ok()); let result = deployment_result.unwrap(); assert_eq!(result.status, DeploymentStatus::Completed); assert_eq!(result.instances_updated, 3); assert!(result.zero_downtime_achieved); assert!(result.deployment_duration_seconds < 10); // Should complete quickly } #[tokio::test] async fn test_rolling_update_respects_batch_size() { // Arrange let config = DeploymentConfig { enable_auto_deployment: true, rolling_update: RollingUpdateConfig { batch_size: 2, // Update 2 instances at a time batch_delay_seconds: 1, health_check_retries: 3, health_check_interval_seconds: 1, }, min_ab_test_confidence: 0.95, ..Default::default() }; let pipeline = DeploymentPipeline::new(config).unwrap(); let model_id = Uuid::new_v4(); let model_path = format!("/tmp/models/{}/model.safetensors", model_id); // Act let start = std::time::Instant::now(); let deployment_result = pipeline .perform_rolling_update(model_id, &model_path, 4) // 4 instances .await; let elapsed = start.elapsed(); // Assert assert!(deployment_result.is_ok()); let result = deployment_result.unwrap(); assert_eq!(result.status, DeploymentStatus::Completed); assert_eq!(result.instances_updated, 4); assert_eq!(result.batches_executed, 2); // 4 instances / batch_size=2 = 2 batches assert!(elapsed.as_secs() >= 1); // Should have delay between batches } // ==================== TEST 3: Health Check Validation ==================== #[tokio::test] async fn test_health_check_validates_model_inference() { // Arrange let config = DeploymentConfig { enable_auto_deployment: true, health_check: HealthCheckConfig { enabled: true, timeout_seconds: 5, max_latency_ms: 100, test_predictions: 10, min_success_rate: 0.95, }, ..Default::default() }; let pipeline = DeploymentPipeline::new(config).unwrap(); let model_id = Uuid::new_v4(); let instance_id = "trading-service-1".to_string(); // Act let health_result = pipeline.run_health_check(model_id, &instance_id).await; // Assert assert!(health_result.is_ok()); let result = health_result.unwrap(); assert!(result.healthy); assert!(result.inference_working); assert!(result.latency_ms < 100.0); assert!(result.success_rate >= 0.95); } #[tokio::test] async fn test_health_check_fails_on_inference_error() { // Arrange let config = DeploymentConfig { enable_auto_deployment: true, health_check: HealthCheckConfig { enabled: true, timeout_seconds: 5, max_latency_ms: 100, test_predictions: 10, min_success_rate: 0.95, }, ..Default::default() }; let pipeline = DeploymentPipeline::new(config).unwrap(); let model_id = Uuid::new_v4(); let instance_id = "trading-service-broken".to_string(); // Simulate broken instance // Act let health_result = pipeline.run_health_check(model_id, &instance_id).await; // Assert assert!(health_result.is_ok()); // Health check runs, but reports unhealthy let result = health_result.unwrap(); assert!(!result.healthy); assert!(!result.inference_working); } #[tokio::test] async fn test_health_check_fails_on_high_latency() { // Arrange let config = DeploymentConfig { enable_auto_deployment: true, health_check: HealthCheckConfig { enabled: true, timeout_seconds: 5, max_latency_ms: 10, // Very strict latency requirement test_predictions: 10, min_success_rate: 0.95, }, ..Default::default() }; let pipeline = DeploymentPipeline::new(config).unwrap(); let model_id = Uuid::new_v4(); let instance_id = "trading-service-slow".to_string(); // Simulate slow instance // Act let health_result = pipeline.run_health_check(model_id, &instance_id).await; // Assert assert!(health_result.is_ok()); let result = health_result.unwrap(); assert!(!result.healthy); assert!(result.latency_ms > 10.0); } // ==================== TEST 4: Rollback on Failure ==================== #[tokio::test] async fn test_rollback_on_health_check_failure() { // Arrange let config = DeploymentConfig { enable_auto_deployment: true, rollback_strategy: RollbackStrategy::Automatic, rollback_on_health_check_failure: true, ..Default::default() }; let pipeline = DeploymentPipeline::new(config).unwrap(); let model_id = Uuid::new_v4(); let previous_model_id = Uuid::new_v4(); let model_path = format!("/tmp/models/{}/model_broken.safetensors", model_id); // Simulate deployment that fails health check let deployment_result = pipeline .deploy_with_rollback(model_id, previous_model_id, &model_path, 2) .await; // Assert assert!(deployment_result.is_ok()); let result = deployment_result.unwrap(); assert_eq!(result.status, DeploymentStatus::RolledBack); assert!(result.rollback_triggered); assert_eq!(result.active_model_id, previous_model_id); // Should revert to previous } #[tokio::test] async fn test_rollback_restores_previous_model() { // Arrange let config = DeploymentConfig { enable_auto_deployment: true, rollback_strategy: RollbackStrategy::Automatic, rollback_on_health_check_failure: true, ..Default::default() }; let pipeline = DeploymentPipeline::new(config).unwrap(); let new_model_id = Uuid::new_v4(); let previous_model_id = Uuid::new_v4(); // Act - Trigger rollback let rollback_result = pipeline .rollback_deployment(new_model_id, previous_model_id) .await; // Assert assert!(rollback_result.is_ok()); let result = rollback_result.unwrap(); assert!(result.rollback_successful); assert_eq!(result.active_model_id, previous_model_id); assert!(result.rollback_duration_seconds < 30); // Should be fast } #[tokio::test] async fn test_manual_rollback_strategy() { // Arrange let config = DeploymentConfig { enable_auto_deployment: true, rollback_strategy: RollbackStrategy::Manual, rollback_on_health_check_failure: false, ..Default::default() }; let pipeline = DeploymentPipeline::new(config).unwrap(); let model_id = Uuid::new_v4(); let previous_model_id = Uuid::new_v4(); let model_path = format!("/tmp/models/{}/model_broken.safetensors", model_id); // Act - Deploy with broken model let deployment_result = pipeline .deploy_with_rollback(model_id, previous_model_id, &model_path, 2) .await; // Assert - Should NOT automatically rollback with Manual strategy assert!(deployment_result.is_ok()); let result = deployment_result.unwrap(); assert_eq!(result.status, DeploymentStatus::Failed); assert!(!result.rollback_triggered); // Manual strategy = no auto-rollback } // ==================== TEST 5: E2E Deployment with Real Model ==================== #[tokio::test] #[ignore = "Run separately: cargo test test_e2e_deployment -- --ignored"] async fn test_e2e_deployment_with_real_model() { // Arrange let config = DeploymentConfig { enable_auto_deployment: true, trigger_on_ab_test_pass: true, rolling_update: RollingUpdateConfig { batch_size: 1, batch_delay_seconds: 2, health_check_retries: 3, health_check_interval_seconds: 1, }, health_check: HealthCheckConfig { enabled: true, timeout_seconds: 10, max_latency_ms: 100, test_predictions: 20, min_success_rate: 0.95, }, rollback_strategy: RollbackStrategy::Automatic, rollback_on_health_check_failure: true, min_ab_test_confidence: 0.95, }; let pipeline = DeploymentPipeline::new(config).unwrap(); // Step 1: Create a real trained model (NO MOCKS) let model_id = Uuid::new_v4(); let model_path = create_real_trained_model(model_id).await.unwrap(); // Step 2: Run A/B test let ab_test_result = create_passing_ab_test_result(model_id); // Step 3: Trigger deployment let trigger_result = pipeline .trigger_deployment_on_ab_test(ab_test_result) .await .unwrap(); assert_eq!(trigger_result.status, DeploymentStatus::Triggered); // Step 4: Perform rolling update let deployment_result = pipeline .perform_rolling_update(model_id, &model_path, 3) .await .unwrap(); assert_eq!(deployment_result.status, DeploymentStatus::Completed); assert_eq!(deployment_result.instances_updated, 3); // Step 5: Verify all instances are healthy for instance_id in &deployment_result.updated_instances { let health = pipeline .run_health_check(model_id, instance_id) .await .unwrap(); assert!(health.healthy); assert!(health.inference_working); } } // ==================== TEST 6: Deployment Monitoring ==================== #[tokio::test] async fn test_deployment_status_tracking() { // Arrange let config = DeploymentConfig::default(); let pipeline = DeploymentPipeline::new(config).unwrap(); let model_id = Uuid::new_v4(); let deployment_id = Uuid::new_v4(); // Act - Start deployment pipeline .start_deployment(deployment_id, model_id) .await .unwrap(); // Query status let status = pipeline.get_deployment_status(deployment_id).await.unwrap(); // Assert assert_eq!(status.deployment_id, deployment_id); assert_eq!(status.model_id, model_id); assert!(matches!( status.status, DeploymentStatus::InProgress | DeploymentStatus::Triggered )); } #[tokio::test] async fn test_deployment_history_tracking() { // Arrange let config = DeploymentConfig::default(); let pipeline = DeploymentPipeline::new(config).unwrap(); // Act - Perform multiple deployments for _ in 0..3 { let model_id = Uuid::new_v4(); let model_path = format!("/tmp/models/{}/model.safetensors", model_id); let _ = pipeline .perform_rolling_update(model_id, &model_path, 1) .await; } // Query history let history = pipeline.get_deployment_history(10).await.unwrap(); // Assert assert!(history.len() >= 3); for deployment in &history { assert!(matches!( deployment.status, DeploymentStatus::Completed | DeploymentStatus::Failed | DeploymentStatus::RolledBack )); } } // ==================== TEST 7: Concurrent Deployment Prevention ==================== #[tokio::test] async fn test_prevents_concurrent_deployments() { // Arrange let config = DeploymentConfig::default(); let pipeline = DeploymentPipeline::new(config).unwrap(); // Act - Start first deployment let model_id_1 = Uuid::new_v4(); let model_path_1 = format!("/tmp/models/{}/model.safetensors", model_id_1); let deployment_1 = pipeline.perform_rolling_update(model_id_1, &model_path_1, 2); // Try to start second deployment concurrently let model_id_2 = Uuid::new_v4(); let model_path_2 = format!("/tmp/models/{}/model.safetensors", model_id_2); let deployment_2 = pipeline.perform_rolling_update(model_id_2, &model_path_2, 2); // Wait for both let (result_1, result_2) = tokio::join!(deployment_1, deployment_2); // Assert - One should succeed, other should be rejected assert!(result_1.is_ok() || result_2.is_ok()); // At least one succeeds assert!(result_1.is_err() || result_2.is_err()); // At least one fails (concurrent rejection) } // ==================== HELPER FUNCTIONS ==================== /// Create A/B test result that passes thresholds fn create_passing_ab_test_result(model_id: Uuid) -> ABTestResult { ABTestResult { experiment_id: Uuid::new_v4(), model_id, control_metrics: GroupMetrics { avg_latency_ms: 50.0, error_rate: 0.01, sharpe_ratio: 1.5, }, treatment_metrics: GroupMetrics { avg_latency_ms: 45.0, // Better latency error_rate: 0.005, // Lower error rate sharpe_ratio: 1.8, // Higher Sharpe ratio }, statistical_significance: 0.99, // High confidence p_value: 0.001, passed: true, } } /// Create A/B test result that fails thresholds fn create_failing_ab_test_result(model_id: Uuid) -> ABTestResult { ABTestResult { experiment_id: Uuid::new_v4(), model_id, control_metrics: GroupMetrics { avg_latency_ms: 50.0, error_rate: 0.01, sharpe_ratio: 1.5, }, treatment_metrics: GroupMetrics { avg_latency_ms: 80.0, // Worse latency error_rate: 0.05, // Higher error rate sharpe_ratio: 1.2, // Lower Sharpe ratio }, statistical_significance: 0.85, // Low confidence p_value: 0.15, passed: false, } } /// Create real trained model for E2E test (NO MOCKS) async fn create_real_trained_model(model_id: Uuid) -> Result { use std::path::PathBuf; let model_dir = PathBuf::from(format!("/tmp/models/{}", model_id)); tokio::fs::create_dir_all(&model_dir).await?; // Create a dummy checkpoint file for testing let checkpoint_path = model_dir.join("model.safetensors"); tokio::fs::write(&checkpoint_path, b"dummy-checkpoint-data").await?; Ok(checkpoint_path.to_string_lossy().to_string()) }