foxhunt/services/ml_training_service/src/deployment_pipeline.rs

//! Automated Production Deployment Pipeline for Trained ML Models
//!
//! **Architecture**:
//! 1. Training completes → Validation passes
//! 2. A/B test passes → Promote to production
//! 3. Rolling update (zero downtime)
//! 4. Health check (model serving correctly)
//! 5. Rollback capability (if health check fails)
//!
//! **Zero Downtime Strategy**:
//! - Rolling update: Update instances in batches
//! - Health check: Verify model inference before routing traffic
//! - Rollback: Automatic revert to previous model on failure

use anyhow::Result;
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::RwLock;
use tokio::time::sleep;
use tracing::{debug, error, info, warn};
use uuid::Uuid;

// ==================== CONFIGURATION ====================

/// Deployment pipeline configuration
#[derive(Debug, Clone)]
pub struct DeploymentConfig {
    /// Enable automatic deployment
    pub enable_auto_deployment: bool,
    /// Trigger deployment on A/B test pass
    pub trigger_on_ab_test_pass: bool,
    /// Minimum A/B test confidence (0.0 - 1.0)
    pub min_ab_test_confidence: f64,
    /// Rolling update configuration
    pub rolling_update: RollingUpdateConfig,
    /// Health check configuration
    pub health_check: HealthCheckConfig,
    /// Rollback strategy
    pub rollback_strategy: RollbackStrategy,
    /// Rollback on health check failure
    pub rollback_on_health_check_failure: bool,
}

impl Default for DeploymentConfig {
    fn default() -> Self {
        Self {
            enable_auto_deployment: true,
            trigger_on_ab_test_pass: true,
            min_ab_test_confidence: 0.95,
            rolling_update: RollingUpdateConfig::default(),
            health_check: HealthCheckConfig::default(),
            rollback_strategy: RollbackStrategy::Automatic,
            rollback_on_health_check_failure: true,
        }
    }
}

/// Rolling update configuration
#[derive(Debug, Clone)]
pub struct RollingUpdateConfig {
    /// Number of instances to update at once
    pub batch_size: usize,
    /// Delay between batches (seconds)
    pub batch_delay_seconds: u64,
    /// Health check retries per instance
    pub health_check_retries: u32,
    /// Health check interval (seconds)
    pub health_check_interval_seconds: u64,
}

impl Default for RollingUpdateConfig {
    fn default() -> Self {
        Self {
            batch_size: 1,
            batch_delay_seconds: 5,
            health_check_retries: 3,
            health_check_interval_seconds: 2,
        }
    }
}

/// Health check configuration
#[derive(Debug, Clone)]
pub struct HealthCheckConfig {
    /// Enable health checks
    pub enabled: bool,
    /// Health check timeout (seconds)
    pub timeout_seconds: u64,
    /// Maximum allowed latency (milliseconds)
    pub max_latency_ms: u64,
    /// Number of test predictions to run
    pub test_predictions: usize,
    /// Minimum success rate (0.0 - 1.0)
    pub min_success_rate: f64,
}

impl Default for HealthCheckConfig {
    fn default() -> Self {
        Self {
            enabled: true,
            timeout_seconds: 10,
            max_latency_ms: 100,
            test_predictions: 10,
            min_success_rate: 0.95,
        }
    }
}

/// Rollback strategy
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RollbackStrategy {
    /// Automatic rollback on failure
    Automatic,
    /// Manual rollback (requires operator intervention)
    Manual,
}

// ==================== RESULT TYPES ====================

/// Deployment status
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum DeploymentStatus {
    /// Deployment triggered
    Triggered,
    /// Deployment in progress
    InProgress,
    /// Deployment completed successfully
    Completed,
    /// Deployment failed
    Failed,
    /// Deployment skipped (e.g., A/B test did not pass)
    Skipped,
    /// Deployment rolled back
    RolledBack,
}

/// Deployment result
#[derive(Debug, Clone)]
pub struct DeploymentResult {
    /// Deployment ID
    pub deployment_id: Uuid,
    /// Model ID being deployed
    pub model_id: Uuid,
    /// Deployment status
    pub status: DeploymentStatus,
    /// Number of instances updated
    pub instances_updated: usize,
    /// Number of batches executed
    pub batches_executed: usize,
    /// Zero downtime achieved
    pub zero_downtime_achieved: bool,
    /// Deployment duration (seconds)
    pub deployment_duration_seconds: u64,
    /// Triggered by A/B test
    pub triggered_by_ab_test: bool,
    /// Rollback triggered
    pub rollback_triggered: bool,
    /// Active model ID after deployment
    pub active_model_id: Uuid,
    /// Updated instance IDs
    pub updated_instances: Vec<String>,
    /// Deployment timestamp
    pub deployed_at: DateTime<Utc>,
    /// Error message (if failed)
    pub error_message: Option<String>,
}

/// Health check result
#[derive(Debug, Clone)]
pub struct HealthCheckResult {
    /// Instance ID
    pub instance_id: String,
    /// Health status
    pub healthy: bool,
    /// Model inference working
    pub inference_working: bool,
    /// Average latency (milliseconds)
    pub latency_ms: f64,
    /// Success rate (0.0 - 1.0)
    pub success_rate: f64,
    /// Error message (if unhealthy)
    pub error_message: Option<String>,
}

/// Rollback result
#[derive(Debug, Clone)]
pub struct RollbackResult {
    /// Rollback successful
    pub rollback_successful: bool,
    /// Active model ID after rollback
    pub active_model_id: Uuid,
    /// Rollback duration (seconds)
    pub rollback_duration_seconds: u64,
    /// Rollback timestamp
    pub rolled_back_at: DateTime<Utc>,
}

/// Deployment status info
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DeploymentStatusInfo {
    /// Deployment ID
    pub deployment_id: Uuid,
    /// Model ID
    pub model_id: Uuid,
    /// Status
    pub status: DeploymentStatus,
    /// Progress (0.0 - 1.0)
    pub progress: f64,
    /// Instances updated
    pub instances_updated: usize,
    /// Total instances
    pub total_instances: usize,
    /// Started at
    pub started_at: DateTime<Utc>,
}

// ==================== DEPLOYMENT PIPELINE ====================

/// Automated production deployment pipeline
pub struct DeploymentPipeline {
    /// Configuration
    config: DeploymentConfig,
    /// Active deployments
    active_deployments: Arc<RwLock<HashMap<Uuid, DeploymentState>>>,
    /// Deployment history
    deployment_history: Arc<RwLock<Vec<DeploymentResult>>>,
    /// Current model per instance
    instance_models: Arc<RwLock<HashMap<String, Uuid>>>,
}

/// Internal deployment state
#[derive(Debug, Clone)]
struct DeploymentState {
    deployment_id: Uuid,
    model_id: Uuid,
    status: DeploymentStatus,
    started_at: DateTime<Utc>,
    instances_updated: usize,
    total_instances: usize,
}

impl DeploymentPipeline {
    /// Create new deployment pipeline
    pub fn new(config: DeploymentConfig) -> Result<Self> {
        info!("Initializing deployment pipeline");

        Ok(Self {
            config,
            active_deployments: Arc::new(RwLock::new(HashMap::new())),
            deployment_history: Arc::new(RwLock::new(Vec::new())),
            instance_models: Arc::new(RwLock::new(HashMap::new())),
        })
    }

    /// Trigger deployment on A/B test completion
    pub async fn trigger_deployment_on_ab_test(
        &self,
        ab_test_result: ABTestResult,
    ) -> Result<DeploymentResult> {
        info!(
            "Evaluating A/B test result for deployment trigger (model: {})",
            ab_test_result.model_id
        );

        // Check if A/B test passed
        if !ab_test_result.passed {
            warn!("A/B test did not pass thresholds, skipping deployment");
            return Ok(DeploymentResult {
                deployment_id: Uuid::new_v4(),
                model_id: ab_test_result.model_id,
                status: DeploymentStatus::Skipped,
                instances_updated: 0,
                batches_executed: 0,
                zero_downtime_achieved: false,
                deployment_duration_seconds: 0,
                triggered_by_ab_test: false,
                rollback_triggered: false,
                active_model_id: ab_test_result.model_id,
                updated_instances: Vec::new(),
                deployed_at: Utc::now(),
                error_message: Some("A/B test failed thresholds".to_string()),
            });
        }

        // Check confidence threshold
        if ab_test_result.statistical_significance < self.config.min_ab_test_confidence {
            warn!(
                "A/B test confidence ({:.2}) below threshold ({:.2}), skipping deployment",
                ab_test_result.statistical_significance, self.config.min_ab_test_confidence
            );
            return Ok(DeploymentResult {
                deployment_id: Uuid::new_v4(),
                model_id: ab_test_result.model_id,
                status: DeploymentStatus::Skipped,
                instances_updated: 0,
                batches_executed: 0,
                zero_downtime_achieved: false,
                deployment_duration_seconds: 0,
                triggered_by_ab_test: false,
                rollback_triggered: false,
                active_model_id: ab_test_result.model_id,
                updated_instances: Vec::new(),
                deployed_at: Utc::now(),
                error_message: Some(format!(
                    "Confidence {:.2} below threshold {:.2}",
                    ab_test_result.statistical_significance, self.config.min_ab_test_confidence
                )),
            });
        }

        // Trigger deployment
        info!(
            "✅ A/B test passed (confidence: {:.2}%), triggering deployment",
            ab_test_result.statistical_significance * 100.0
        );

        Ok(DeploymentResult {
            deployment_id: Uuid::new_v4(),
            model_id: ab_test_result.model_id,
            status: DeploymentStatus::Triggered,
            instances_updated: 0,
            batches_executed: 0,
            zero_downtime_achieved: false,
            deployment_duration_seconds: 0,
            triggered_by_ab_test: true,
            rollback_triggered: false,
            active_model_id: ab_test_result.model_id,
            updated_instances: Vec::new(),
            deployed_at: Utc::now(),
            error_message: None,
        })
    }

    /// Perform rolling update with zero downtime
    pub async fn perform_rolling_update(
        &self,
        model_id: Uuid,
        model_path: &str,
        total_instances: usize,
    ) -> Result<DeploymentResult> {
        let deployment_id = Uuid::new_v4();
        let start_time = std::time::Instant::now();

        info!(
            "🚀 Starting rolling update: deployment_id={}, model_id={}, instances={}",
            deployment_id, model_id, total_instances
        );

        // Register deployment
        self.start_deployment(deployment_id, model_id).await?;

        let batch_size = self.config.rolling_update.batch_size;
        let num_batches = total_instances.div_ceil(batch_size);
        let mut updated_instances = Vec::new();

        // Process instances in batches
        for batch_idx in 0..num_batches {
            let start_idx = batch_idx * batch_size;
            let end_idx = ((batch_idx + 1) * batch_size).min(total_instances);
            let batch_instances: Vec<String> = (start_idx..end_idx)
                .map(|i| format!("trading-service-{}", i + 1))
                .collect();

            info!(
                "📦 Processing batch {}/{}: {} instances",
                batch_idx + 1,
                num_batches,
                batch_instances.len()
            );

            // Update instances in this batch
            for instance_id in &batch_instances {
                debug!("Updating instance: {}", instance_id);

                // Simulate model loading (in production: gRPC call to TradingService)
                self.load_model_on_instance(instance_id, model_id, model_path)
                    .await?;

                // Run health check
                if self.config.health_check.enabled {
                    let health = self.run_health_check(model_id, instance_id).await?;
                    if !health.healthy {
                        error!(
                            "Health check failed for instance {}: {:?}",
                            instance_id, health
                        );
                        return Ok(DeploymentResult {
                            deployment_id,
                            model_id,
                            status: DeploymentStatus::Failed,
                            instances_updated: updated_instances.len(),
                            batches_executed: batch_idx + 1,
                            zero_downtime_achieved: false,
                            deployment_duration_seconds: start_time.elapsed().as_secs(),
                            triggered_by_ab_test: false,
                            rollback_triggered: false,
                            active_model_id: model_id,
                            updated_instances,
                            deployed_at: Utc::now(),
                            error_message: Some(format!(
                                "Health check failed for instance {}",
                                instance_id
                            )),
                        });
                    }
                }

                updated_instances.push(instance_id.clone());
                debug!("✅ Instance {} updated successfully", instance_id);
            }

            // Delay between batches (except for last batch)
            if batch_idx < num_batches - 1 {
                debug!(
                    "⏳ Waiting {} seconds before next batch",
                    self.config.rolling_update.batch_delay_seconds
                );
                sleep(Duration::from_secs(
                    self.config.rolling_update.batch_delay_seconds,
                ))
                .await;
            }
        }

        let duration = start_time.elapsed();
        info!(
            "✅ Rolling update completed: {} instances updated in {:.2}s",
            updated_instances.len(),
            duration.as_secs_f64()
        );

        Ok(DeploymentResult {
            deployment_id,
            model_id,
            status: DeploymentStatus::Completed,
            instances_updated: updated_instances.len(),
            batches_executed: num_batches,
            zero_downtime_achieved: true,
            deployment_duration_seconds: duration.as_secs(),
            triggered_by_ab_test: false,
            rollback_triggered: false,
            active_model_id: model_id,
            updated_instances,
            deployed_at: Utc::now(),
            error_message: None,
        })
    }

    /// Deploy with automatic rollback on failure
    pub async fn deploy_with_rollback(
        &self,
        model_id: Uuid,
        previous_model_id: Uuid,
        model_path: &str,
        total_instances: usize,
    ) -> Result<DeploymentResult> {
        info!(
            "Deploying model {} with rollback capability (previous: {})",
            model_id, previous_model_id
        );

        // Attempt deployment
        let deployment_result = self
            .perform_rolling_update(model_id, model_path, total_instances)
            .await?;

        // Check if deployment failed
        if deployment_result.status == DeploymentStatus::Failed
            && self.config.rollback_on_health_check_failure
                && self.config.rollback_strategy == RollbackStrategy::Automatic
            {
                error!("Deployment failed, triggering automatic rollback");

                // Perform rollback
                let rollback_result = self
                    .rollback_deployment(model_id, previous_model_id)
                    .await?;

                return Ok(DeploymentResult {
                    rollback_triggered: true,
                    status: DeploymentStatus::RolledBack,
                    active_model_id: rollback_result.active_model_id,
                    ..deployment_result
                });
            }

        Ok(deployment_result)
    }

    /// Run health check on deployed model
    pub async fn run_health_check(
        &self,
        model_id: Uuid,
        instance_id: &str,
    ) -> Result<HealthCheckResult> {
        debug!(
            "Running health check: model={}, instance={}",
            model_id, instance_id
        );

        // Simulate health check based on instance name
        let is_broken = instance_id.contains("broken");
        let is_slow = instance_id.contains("slow");

        if is_broken {
            // Simulate broken instance
            return Ok(HealthCheckResult {
                instance_id: instance_id.to_string(),
                healthy: false,
                inference_working: false,
                latency_ms: 0.0,
                success_rate: 0.0,
                error_message: Some("Model inference failed".to_string()),
            });
        }

        if is_slow {
            // Simulate slow instance
            return Ok(HealthCheckResult {
                instance_id: instance_id.to_string(),
                healthy: false,
                inference_working: true,
                latency_ms: 500.0, // High latency
                success_rate: 1.0,
                error_message: Some(format!(
                    "Latency {:.2}ms exceeds threshold {}ms",
                    500.0, self.config.health_check.max_latency_ms
                )),
            });
        }

        // Simulate successful health check
        let latency_ms = 45.0;
        let success_rate = 0.98;

        Ok(HealthCheckResult {
            instance_id: instance_id.to_string(),
            healthy: latency_ms <= self.config.health_check.max_latency_ms as f64
                && success_rate >= self.config.health_check.min_success_rate,
            inference_working: true,
            latency_ms,
            success_rate,
            error_message: None,
        })
    }

    /// Rollback to previous model
    pub async fn rollback_deployment(
        &self,
        _current_model_id: Uuid,
        previous_model_id: Uuid,
    ) -> Result<RollbackResult> {
        let start_time = std::time::Instant::now();

        info!("🔄 Rolling back to previous model: {}", previous_model_id);

        // In production: Issue rollback commands to all TradingService instances
        // For now: Simulate fast rollback
        sleep(Duration::from_millis(500)).await;

        let duration = start_time.elapsed();

        info!("✅ Rollback completed in {:.2}s", duration.as_secs_f64());

        Ok(RollbackResult {
            rollback_successful: true,
            active_model_id: previous_model_id,
            rollback_duration_seconds: duration.as_secs(),
            rolled_back_at: Utc::now(),
        })
    }

    /// Start a deployment (internal tracking)
    pub async fn start_deployment(&self, deployment_id: Uuid, model_id: Uuid) -> Result<()> {
        let mut active = self.active_deployments.write().await;

        // Check for concurrent deployments
        if !active.is_empty() {
            return Err(anyhow::anyhow!("Deployment already in progress"));
        }

        active.insert(
            deployment_id,
            DeploymentState {
                deployment_id,
                model_id,
                status: DeploymentStatus::InProgress,
                started_at: Utc::now(),
                instances_updated: 0,
                total_instances: 0,
            },
        );

        Ok(())
    }

    /// Get deployment status
    pub async fn get_deployment_status(&self, deployment_id: Uuid) -> Result<DeploymentStatusInfo> {
        let active = self.active_deployments.read().await;

        let state = active
            .get(&deployment_id)
            .ok_or_else(|| anyhow::anyhow!("Deployment not found"))?;

        Ok(DeploymentStatusInfo {
            deployment_id: state.deployment_id,
            model_id: state.model_id,
            status: state.status.clone(),
            progress: if state.total_instances > 0 {
                state.instances_updated as f64 / state.total_instances as f64
            } else {
                0.0
            },
            instances_updated: state.instances_updated,
            total_instances: state.total_instances,
            started_at: state.started_at,
        })
    }

    /// Get deployment history
    pub async fn get_deployment_history(&self, limit: usize) -> Result<Vec<DeploymentResult>> {
        let history = self.deployment_history.read().await;
        Ok(history.iter().rev().take(limit).cloned().collect())
    }

    /// Load model on instance (simulate gRPC call)
    async fn load_model_on_instance(
        &self,
        instance_id: &str,
        model_id: Uuid,
        _model_path: &str,
    ) -> Result<()> {
        debug!("Loading model {} on instance {}", model_id, instance_id);

        // Simulate model loading delay
        sleep(Duration::from_millis(100)).await;

        // Update instance model mapping
        let mut instance_models = self.instance_models.write().await;
        instance_models.insert(instance_id.to_string(), model_id);

        Ok(())
    }
}

// ==================== EXTERNAL DATA STRUCTURES ====================

/// A/B test result (matches ml/src/deployment/ab_testing.rs)
#[derive(Debug, Clone)]
pub struct ABTestResult {
    pub experiment_id: Uuid,
    pub model_id: Uuid,
    pub control_metrics: GroupMetrics,
    pub treatment_metrics: GroupMetrics,
    pub statistical_significance: f64,
    pub p_value: f64,
    pub passed: bool,
}

#[derive(Debug, Clone)]
pub struct GroupMetrics {
    pub avg_latency_ms: f64,
    pub error_rate: f64,
    pub sharpe_ratio: f64,
}

#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used)]
mod tests {
    use super::*;

    #[test]
    fn test_deployment_config_default() {
        let config = DeploymentConfig::default();
        assert!(config.enable_auto_deployment);
        assert_eq!(config.min_ab_test_confidence, 0.95);
        assert_eq!(config.rolling_update.batch_size, 1);
    }

    #[tokio::test]
    async fn test_pipeline_creation() {
        let config = DeploymentConfig::default();
        let pipeline = DeploymentPipeline::new(config);
        assert!(pipeline.is_ok());
    }

    #[tokio::test]
    async fn test_concurrent_deployment_prevention() {
        let config = DeploymentConfig::default();
        let pipeline = DeploymentPipeline::new(config).unwrap();

        let deployment_id_1 = Uuid::new_v4();
        let deployment_id_2 = Uuid::new_v4();
        let model_id = Uuid::new_v4();

        // Start first deployment
        let result1 = pipeline.start_deployment(deployment_id_1, model_id).await;
        assert!(result1.is_ok());

        // Try to start second deployment (should fail)
        let result2 = pipeline.start_deployment(deployment_id_2, model_id).await;
        assert!(result2.is_err());
    }
}