foxhunt/crates/ml/src/deployment/validation.rs

//! Multi-Stage Validation Pipeline for Model Deployments
//!
//! This module implements a comprehensive validation pipeline that validates
//! models through multiple stages before production deployment.

use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime};

use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use tokio::sync::{RwLock, Mutex};
use uuid::Uuid;

use crate::{MLError, MLResult, ModelType, Features, ModelPrediction, MLModel};
use super::{ModelVersion, DeploymentStatus, PerformanceBaseline};

/// Validation pipeline configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ValidationConfig {
    /// Enabled validation stages
    pub enabled_stages: Vec<ValidationStage>,
    /// Validation timeout per stage
    pub stage_timeout: Duration,
    /// Total validation timeout
    pub total_timeout: Duration,
    /// Parallel validation (where possible)
    pub parallel_execution: bool,
    /// Fail fast on first error
    pub fail_fast: bool,
    /// Performance requirements
    pub performance_requirements: PerformanceRequirements,
    /// Security validation settings
    pub security_validation: SecurityValidationConfig,
    /// Custom validation rules
    pub custom_validations: Vec<CustomValidationRule>,
}

impl Default for ValidationConfig {
    fn default() -> Self {
        Self {
            enabled_stages: vec![
                ValidationStage::Syntax,
                ValidationStage::UnitTests,
                ValidationStage::IntegrationTests,
                ValidationStage::PerformanceTests,
                ValidationStage::SecurityTests,
                ValidationStage::CanaryDeployment,
            ],
            stage_timeout: Duration::from_secs(300), // 5 minutes per stage
            total_timeout: Duration::from_secs(1800), // 30 minutes total
            parallel_execution: true,
            fail_fast: true,
            performance_requirements: PerformanceRequirements::default(),
            security_validation: SecurityValidationConfig::default(),
            custom_validations: Vec::new(),
        }
    }
}

/// Validation stages in the pipeline
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum ValidationStage {
    /// Syntax and format validation
    Syntax,
    /// Unit tests for model functionality
    UnitTests,
    /// Integration tests with other components
    IntegrationTests,
    /// Performance benchmarking
    PerformanceTests,
    /// Security vulnerability scanning
    SecurityTests,
    /// Canary deployment validation
    CanaryDeployment,
    /// Custom validation stage
    Custom(String),
}

impl ValidationStage {
    /// Get stage name as string
    pub fn name(&self) -> &str {
        match self {
            ValidationStage::Syntax => "syntax",
            ValidationStage::UnitTests => "unit_tests",
            ValidationStage::IntegrationTests => "integration_tests",
            ValidationStage::PerformanceTests => "performance_tests",
            ValidationStage::SecurityTests => "security_tests",
            ValidationStage::CanaryDeployment => "canary_deployment",
            ValidationStage::Custom(name) => name,
        }
    }

    /// Get stage execution order priority
    pub fn priority(&self) -> u8 {
        match self {
            ValidationStage::Syntax => 1,
            ValidationStage::UnitTests => 2,
            ValidationStage::IntegrationTests => 3,
            ValidationStage::SecurityTests => 4,
            ValidationStage::PerformanceTests => 5,
            ValidationStage::CanaryDeployment => 6,
            ValidationStage::Custom(_) => 7,
        }
    }

    /// Check if stage can run in parallel with others
    pub fn can_run_parallel(&self) -> bool {
        match self {
            ValidationStage::Syntax => true,
            ValidationStage::UnitTests => true,
            ValidationStage::SecurityTests => true,
            ValidationStage::IntegrationTests => false, // May affect other stages
            ValidationStage::PerformanceTests => false, // Resource intensive
            ValidationStage::CanaryDeployment => false, // Must be last
            ValidationStage::Custom(_) => false, // Conservative default
        }
    }
}

/// Performance requirements for validation
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PerformanceRequirements {
    /// Maximum average latency in microseconds
    pub max_avg_latency_us: u64,
    /// Maximum 95th percentile latency in microseconds
    pub max_p95_latency_us: u64,
    /// Maximum 99th percentile latency in microseconds
    pub max_p99_latency_us: u64,
    /// Minimum throughput in predictions per second
    pub min_throughput_pps: u32,
    /// Maximum memory usage in MB
    pub max_memory_usage_mb: u64,
    /// Maximum CPU utilization percentage
    pub max_cpu_utilization: f32,
    /// Maximum error rate (0.0 to 1.0)
    pub max_error_rate: f32,
    /// Minimum accuracy score (0.0 to 1.0)
    pub min_accuracy_score: f32,
}

impl Default for PerformanceRequirements {
    fn default() -> Self {
        Self {
            max_avg_latency_us: 100,   // 100 microseconds
            max_p95_latency_us: 200,   // 200 microseconds
            max_p99_latency_us: 500,   // 500 microseconds
            min_throughput_pps: 10000, // 10K predictions per second
            max_memory_usage_mb: 1024, // 1GB
            max_cpu_utilization: 80.0, // 80%
            max_error_rate: 0.01,      // 1%
            min_accuracy_score: 0.8,   // 80%
        }
    }
}

/// Security validation configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SecurityValidationConfig {
    /// Enable vulnerability scanning
    pub enable_vulnerability_scan: bool,
    /// Enable dependency security check
    pub enable_dependency_check: bool,
    /// Enable model poisoning detection
    pub enable_poisoning_detection: bool,
    /// Enable adversarial robustness testing
    pub enable_adversarial_testing: bool,
    /// Security scan timeout
    pub scan_timeout: Duration,
}

impl Default for SecurityValidationConfig {
    fn default() -> Self {
        Self {
            enable_vulnerability_scan: true,
            enable_dependency_check: true,
            enable_poisoning_detection: true,
            enable_adversarial_testing: false, // Computationally expensive
            scan_timeout: Duration::from_secs(600), // 10 minutes
        }
    }
}

/// Custom validation rule
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CustomValidationRule {
    /// Rule name
    pub name: String,
    /// Rule description
    pub description: String,
    /// Rule implementation (as a validation function identifier)
    pub validator_id: String,
    /// Rule parameters
    pub parameters: HashMap<String, String>,
    /// Whether the rule is required or optional
    pub required: bool,
}

/// Validation result for a single stage
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ValidationResult {
    /// Validation stage
    pub stage: ValidationStage,
    /// Validation status
    pub status: ValidationStatus,
    /// Start time
    pub start_time: SystemTime,
    /// End time
    pub end_time: Option<SystemTime>,
    /// Duration
    pub duration: Option<Duration>,
    /// Success flag
    pub success: bool,
    /// Error message (if failed)
    pub error_message: Option<String>,
    /// Validation metrics
    pub metrics: ValidationMetrics,
    /// Stage-specific results
    pub stage_results: StageResults,
}

/// Validation status
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum ValidationStatus {
    /// Validation is pending
    Pending,
    /// Validation is running
    Running,
    /// Validation completed successfully
    Passed,
    /// Validation failed
    Failed,
    /// Validation was skipped
    Skipped,
    /// Validation timed out
    TimedOut,
}

/// Validation metrics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ValidationMetrics {
    /// Number of tests run
    pub tests_run: u32,
    /// Number of tests passed
    pub tests_passed: u32,
    /// Number of tests failed
    pub tests_failed: u32,
    /// Coverage percentage (if applicable)
    pub coverage_percentage: Option<f32>,
    /// Performance metrics
    pub performance_metrics: Option<PerformanceBaseline>,
    /// Security findings
    pub security_findings: Vec<SecurityFinding>,
    /// Custom metrics
    pub custom_metrics: HashMap<String, f64>,
}

impl Default for ValidationMetrics {
    fn default() -> Self {
        Self {
            tests_run: 0,
            tests_passed: 0,
            tests_failed: 0,
            coverage_percentage: None,
            performance_metrics: None,
            security_findings: Vec::new(),
            custom_metrics: HashMap::new(),
        }
    }
}

/// Stage-specific validation results
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum StageResults {
    /// Syntax validation results
    Syntax(SyntaxValidationResults),
    /// Unit test results
    UnitTests(UnitTestResults),
    /// Integration test results
    IntegrationTests(IntegrationTestResults),
    /// Performance test results
    PerformanceTests(PerformanceTestResults),
    /// Security test results
    SecurityTests(SecurityTestResults),
    /// Canary deployment results
    CanaryDeployment(CanaryDeploymentResults),
    /// Custom validation results
    Custom(CustomValidationResults),
}

/// Syntax validation results
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SyntaxValidationResults {
    /// Model format is valid
    pub format_valid: bool,
    /// Model schema validation
    pub schema_valid: bool,
    /// Checksum validation
    pub checksum_valid: bool,
    /// File integrity check
    pub integrity_valid: bool,
    /// Syntax errors found
    pub syntax_errors: Vec<String>,
}

/// Unit test results
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct UnitTestResults {
    /// Individual test results
    pub test_results: Vec<TestCase>,
    /// Overall test suite success
    pub suite_success: bool,
    /// Code coverage metrics
    pub coverage: Option<CoverageMetrics>,
}

/// Individual test case result
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TestCase {
    /// Test name
    pub name: String,
    /// Test status
    pub status: TestStatus,
    /// Test duration
    pub duration: Duration,
    /// Error message (if failed)
    pub error_message: Option<String>,
    /// Test output
    pub output: Option<String>,
}

/// Test status
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum TestStatus {
    /// Test passed
    Passed,
    /// Test failed
    Failed,
    /// Test was skipped
    Skipped,
    /// Test timed out
    TimedOut,
}

/// Code coverage metrics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CoverageMetrics {
    /// Line coverage percentage
    pub line_coverage: f32,
    /// Branch coverage percentage
    pub branch_coverage: f32,
    /// Function coverage percentage
    pub function_coverage: f32,
}

/// Integration test results
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IntegrationTestResults {
    /// End-to-end test results
    pub e2e_tests: Vec<TestCase>,
    /// API compatibility tests
    pub api_compatibility: bool,
    /// Data pipeline tests
    pub data_pipeline_tests: Vec<TestCase>,
    /// Service integration tests
    pub service_integration_tests: Vec<TestCase>,
}

/// Performance test results
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PerformanceTestResults {
    /// Latency benchmark results
    pub latency_benchmarks: LatencyBenchmarks,
    /// Throughput benchmark results
    pub throughput_benchmarks: ThroughputBenchmarks,
    /// Memory usage benchmarks
    pub memory_benchmarks: MemoryBenchmarks,
    /// Load test results
    pub load_test_results: LoadTestResults,
    /// Stress test results
    pub stress_test_results: Option<StressTestResults>,
}

/// Latency benchmark results
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LatencyBenchmarks {
    /// Average latency in microseconds
    pub avg_latency_us: f64,
    /// Median latency in microseconds
    pub median_latency_us: f64,
    /// 95th percentile latency
    pub p95_latency_us: f64,
    /// 99th percentile latency
    pub p99_latency_us: f64,
    /// 99.9th percentile latency
    pub p999_latency_us: f64,
    /// Maximum latency observed
    pub max_latency_us: f64,
    /// Latency distribution
    pub latency_distribution: Vec<(f64, u32)>, // (latency_bucket, count)
}

/// Throughput benchmark results
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ThroughputBenchmarks {
    /// Requests per second
    pub requests_per_second: f64,
    /// Predictions per second
    pub predictions_per_second: f64,
    /// Peak throughput achieved
    pub peak_throughput: f64,
    /// Sustained throughput
    pub sustained_throughput: f64,
}

/// Memory benchmark results
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryBenchmarks {
    /// Peak memory usage in MB
    pub peak_memory_mb: f64,
    /// Average memory usage in MB
    pub avg_memory_mb: f64,
    /// Memory usage growth rate
    pub memory_growth_rate: f64,
    /// Memory leaks detected
    pub memory_leaks_detected: bool,
}

/// Load test results
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LoadTestResults {
    /// Test duration
    pub test_duration: Duration,
    /// Target load achieved
    pub target_load_achieved: bool,
    /// Error rate during load test
    pub error_rate: f32,
    /// Response time degradation
    pub response_time_degradation: f32,
}

/// Stress test results
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StressTestResults {
    /// Breaking point (requests per second)
    pub breaking_point_rps: Option<f64>,
    /// Recovery time after stress
    pub recovery_time: Duration,
    /// System stability during stress
    pub stability_score: f32,
}

/// Security test results
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SecurityTestResults {
    /// Vulnerability scan results
    pub vulnerability_scan: VulnerabilityScanResults,
    /// Dependency security check results
    pub dependency_check: DependencyCheckResults,
    /// Model poisoning detection results
    pub poisoning_detection: Option<PoisoningDetectionResults>,
    /// Adversarial robustness test results
    pub adversarial_testing: Option<AdversarialTestResults>,
}

/// Vulnerability scan results
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VulnerabilityScanResults {
    /// Total vulnerabilities found
    pub total_vulnerabilities: u32,
    /// Critical vulnerabilities
    pub critical_vulnerabilities: u32,
    /// High severity vulnerabilities
    pub high_vulnerabilities: u32,
    /// Medium severity vulnerabilities
    pub medium_vulnerabilities: u32,
    /// Low severity vulnerabilities
    pub low_vulnerabilities: u32,
    /// Detailed findings
    pub findings: Vec<SecurityFinding>,
}

/// Dependency security check results
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DependencyCheckResults {
    /// Total dependencies checked
    pub total_dependencies: u32,
    /// Vulnerable dependencies found
    pub vulnerable_dependencies: u32,
    /// Outdated dependencies
    pub outdated_dependencies: u32,
    /// Security advisories
    pub security_advisories: Vec<SecurityAdvisory>,
}

/// Model poisoning detection results
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PoisoningDetectionResults {
    /// Poisoning detected flag
    pub poisoning_detected: bool,
    /// Confidence score of detection
    pub detection_confidence: f32,
    /// Poisoning type detected
    pub poisoning_type: Option<String>,
    /// Affected model components
    pub affected_components: Vec<String>,
}

/// Adversarial robustness test results
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AdversarialTestResults {
    /// Robustness score (0.0 to 1.0)
    pub robustness_score: f32,
    /// Successful adversarial attacks
    pub successful_attacks: u32,
    /// Total adversarial tests
    pub total_tests: u32,
    /// Attack success rate
    pub attack_success_rate: f32,
}

/// Security finding
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SecurityFinding {
    /// Finding ID
    pub id: String,
    /// Severity level
    pub severity: SecuritySeverity,
    /// Finding title
    pub title: String,
    /// Finding description
    pub description: String,
    /// Affected component
    pub component: String,
    /// Recommendation for fix
    pub recommendation: String,
    /// CVE identifier (if applicable)
    pub cve_id: Option<String>,
}

/// Security severity levels
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
pub enum SecuritySeverity {
    /// Low severity
    Low,
    /// Medium severity
    Medium,
    /// High severity
    High,
    /// Critical severity
    Critical,
}

/// Security advisory
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SecurityAdvisory {
    /// Advisory ID
    pub id: String,
    /// Affected package
    pub package: String,
    /// Vulnerable versions
    pub vulnerable_versions: String,
    /// Patched versions
    pub patched_versions: String,
    /// Advisory summary
    pub summary: String,
    /// Advisory URL
    pub url: Option<String>,
}

/// Canary deployment results
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CanaryDeploymentResults {
    /// Canary traffic percentage
    pub canary_percentage: f32,
    /// Canary duration
    pub canary_duration: Duration,
    /// Canary success rate
    pub success_rate: f32,
    /// Performance comparison with production
    pub performance_comparison: PerformanceComparison,
    /// Error rate comparison
    pub error_rate_comparison: f32,
    /// User feedback (if available)
    pub user_feedback: Option<UserFeedback>,
}

/// Performance comparison between canary and production
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PerformanceComparison {
    /// Latency difference (positive means canary is slower)
    pub latency_difference_percent: f32,
    /// Throughput difference (positive means canary is faster)
    pub throughput_difference_percent: f32,
    /// Memory usage difference (positive means canary uses more)
    pub memory_difference_percent: f32,
    /// Overall performance score
    pub overall_score: f32,
}

/// User feedback for canary deployment
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct UserFeedback {
    /// Total feedback entries
    pub total_feedback: u32,
    /// Positive feedback count
    pub positive_feedback: u32,
    /// Negative feedback count
    pub negative_feedback: u32,
    /// Average rating (1.0 to 5.0)
    pub average_rating: f32,
    /// Feedback comments
    pub comments: Vec<String>,
}

/// Custom validation results
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CustomValidationResults {
    /// Validation name
    pub validation_name: String,
    /// Custom result data
    pub result_data: HashMap<String, serde_json::Value>,
    /// Success flag
    pub success: bool,
    /// Custom metrics
    pub metrics: HashMap<String, f64>,
}

/// Validation pipeline executor
pub struct ValidationPipeline {
    /// Pipeline configuration
    config: ValidationConfig,
    /// Validation stages
    stages: Vec<Box<dyn ValidationStageExecutor>>,
    /// Pipeline state
    state: Arc<RwLock<PipelineState>>,
}

/// Pipeline execution state
#[derive(Debug, Clone)]
struct PipelineState {
    /// Current stage being executed
    current_stage: Option<ValidationStage>,
    /// Stage results
    stage_results: HashMap<ValidationStage, ValidationResult>,
    /// Pipeline start time
    start_time: SystemTime,
    /// Pipeline status
    status: PipelineStatus,
}

/// Pipeline execution status
#[derive(Debug, Clone, PartialEq, Eq)]
enum PipelineStatus {
    /// Pipeline is idle
    Idle,
    /// Pipeline is running
    Running,
    /// Pipeline completed successfully
    Completed,
    /// Pipeline failed
    Failed,
    /// Pipeline was cancelled
    Cancelled,
}

impl ValidationPipeline {
    /// Create new validation pipeline
    pub fn new(config: ValidationConfig) -> Self {
        let mut stages: Vec<Box<dyn ValidationStageExecutor>> = Vec::new();
        
        // Add default stage implementations
        for stage in &config.enabled_stages {
            match stage {
                ValidationStage::Syntax => stages.push(Box::new(SyntaxValidator::new())),
                ValidationStage::UnitTests => stages.push(Box::new(UnitTestValidator::new())),
                ValidationStage::IntegrationTests => stages.push(Box::new(IntegrationTestValidator::new())),
                ValidationStage::PerformanceTests => stages.push(Box::new(PerformanceTestValidator::new())),
                ValidationStage::SecurityTests => stages.push(Box::new(SecurityTestValidator::new())),
                ValidationStage::CanaryDeployment => stages.push(Box::new(CanaryDeploymentValidator::new())),
                ValidationStage::Custom(name) => {
                    // Custom validators would be registered separately
                    tracing::warn!("Custom validation stage '{}' not implemented", name);
                }
            }
        }

        let state = PipelineState {
            current_stage: None,
            stage_results: HashMap::new(),
            start_time: SystemTime::now(),
            status: PipelineStatus::Idle,
        };

        Self {
            config,
            stages,
            state: Arc::new(RwLock::new(state)),
        }
    }

    /// Execute validation pipeline
    pub async fn execute(&self, model: Arc<dyn MLModel>, version: &ModelVersion) -> MLResult<PipelineExecutionResult> {
        let execution_start = Instant::now();
        
        // Update pipeline state
        {
            let mut state = self.state.write().await;
            state.status = PipelineStatus::Running;
            state.start_time = SystemTime::now();
            state.stage_results.clear();
        }

        let mut stage_results = HashMap::new();
        let mut overall_success = true;

        // Execute stages based on configuration
        if self.config.parallel_execution {
            // Execute parallelizable stages in parallel
            overall_success = self.execute_parallel_stages(&model, version, &mut stage_results).await?;
        } else {
            // Execute stages sequentially
            overall_success = self.execute_sequential_stages(&model, version, &mut stage_results).await?;
        }

        // Update final state
        {
            let mut state = self.state.write().await;
            state.status = if overall_success { PipelineStatus::Completed } else { PipelineStatus::Failed };
            state.stage_results = stage_results.clone();
        }

        let execution_duration = execution_start.elapsed();

        Ok(PipelineExecutionResult {
            success: overall_success,
            execution_duration,
            stage_results,
            summary: self.generate_execution_summary(&stage_results, overall_success).await,
        })
    }

    /// Execute stages in parallel where possible
    async fn execute_parallel_stages(
        &self,
        model: &Arc<dyn MLModel>,
        version: &ModelVersion,
        stage_results: &mut HashMap<ValidationStage, ValidationResult>,
    ) -> MLResult<bool> {
        use futures::future::join_all;

        // Group stages by execution order and parallelizability
        let mut sequential_stages = Vec::new();
        let mut parallel_stages = Vec::new();

        for stage in &self.stages {
            let stage_type = stage.stage_type();
            if stage_type.can_run_parallel() {
                parallel_stages.push(stage);
            } else {
                sequential_stages.push(stage);
            }
        }

        let mut overall_success = true;

        // Execute parallel stages first
        if !parallel_stages.is_empty() {
            let parallel_futures = parallel_stages.into_iter().map(|stage| {
                let model = model.clone();
                let version = version.clone();
                async move {
                    stage.execute(model, &version).await
                }
            });

            let parallel_results = join_all(parallel_futures).await;
            
            for result in parallel_results {
                match result {
                    Ok(validation_result) => {
                        let success = validation_result.success;
                        let stage = validation_result.stage;
                        stage_results.insert(stage, validation_result);
                        
                        if !success {
                            overall_success = false;
                            if self.config.fail_fast {
                                return Ok(false);
                            }
                        }
                    }
                    Err(e) => {
                        tracing::error!("Parallel stage execution failed: {}", e);
                        overall_success = false;
                        if self.config.fail_fast {
                            return Ok(false);
                        }
                    }
                }
            }
        }

        // Execute sequential stages
        for stage in sequential_stages {
            let result = stage.execute(model.clone(), version).await?;
            let success = result.success;
            let stage_type = result.stage;
            
            stage_results.insert(stage_type, result);
            
            if !success {
                overall_success = false;
                if self.config.fail_fast {
                    break;
                }
            }
        }

        Ok(overall_success)
    }

    /// Execute stages sequentially
    async fn execute_sequential_stages(
        &self,
        model: &Arc<dyn MLModel>,
        version: &ModelVersion,
        stage_results: &mut HashMap<ValidationStage, ValidationResult>,
    ) -> MLResult<bool> {
        let mut overall_success = true;

        // Sort stages by priority
        let mut sorted_stages: Vec<&Box<dyn ValidationStageExecutor>> = self.stages.iter().collect();
        sorted_stages.sort_by_key(|stage| stage.stage_type().priority());

        for stage in sorted_stages {
            {
                let mut state = self.state.write().await;
                state.current_stage = Some(stage.stage_type());
            }

            let result = stage.execute(model.clone(), version).await?;
            let success = result.success;
            let stage_type = result.stage;
            
            stage_results.insert(stage_type, result);
            
            if !success {
                overall_success = false;
                if self.config.fail_fast {
                    break;
                }
            }
        }

        Ok(overall_success)
    }

    /// Generate execution summary
    async fn generate_execution_summary(
        &self,
        stage_results: &HashMap<ValidationStage, ValidationResult>,
        overall_success: bool,
    ) -> ValidationSummary {
        let total_stages = stage_results.len();
        let passed_stages = stage_results.values().filter(|r| r.success).count();
        let failed_stages = total_stages - passed_stages;

        let total_duration = stage_results.values()
            .filter_map(|r| r.duration)
            .fold(Duration::new(0, 0), |acc, d| acc + d);

        ValidationSummary {
            overall_success,
            total_stages,
            passed_stages,
            failed_stages,
            total_duration,
            critical_issues: self.count_critical_issues(stage_results),
            recommendations: self.generate_recommendations(stage_results).await,
        }
    }

    /// Count critical issues across all stages
    fn count_critical_issues(&self, stage_results: &HashMap<ValidationStage, ValidationResult>) -> u32 {
        stage_results.values()
            .map(|result| {
                result.metrics.security_findings.iter()
                    .filter(|finding| finding.severity == SecuritySeverity::Critical)
                    .count() as u32
            })
            .sum()
    }

    /// Generate recommendations based on validation results
    async fn generate_recommendations(&self, stage_results: &HashMap<ValidationStage, ValidationResult>) -> Vec<String> {
        let mut recommendations = Vec::new();

        for (stage, result) in stage_results {
            if !result.success {
                recommendations.push(format!(
                    "Fix issues in {} stage: {}",
                    stage.name(),
                    result.error_message.as_deref().unwrap_or("Unknown error")
                ));
            }

            // Stage-specific recommendations
            match stage {
                ValidationStage::PerformanceTests => {
                    if let Some(ref perf_metrics) = result.metrics.performance_metrics {
                        if perf_metrics.avg_latency_us > self.config.performance_requirements.max_avg_latency_us as f64 {
                            recommendations.push("Consider optimizing model inference latency".to_owned());
                        }
                    }
                }
                ValidationStage::SecurityTests => {
                    let critical_findings = result.metrics.security_findings.iter()
                        .filter(|f| f.severity == SecuritySeverity::Critical)
                        .count();
                    if critical_findings > 0 {
                        recommendations.push(format!("Address {} critical security findings before deployment", critical_findings));
                    }
                }
                _ => {}
            }
        }

        recommendations
    }

    /// Get current pipeline status
    pub async fn get_status(&self) -> PipelineStatusInfo {
        let state = self.state.read().await;
        PipelineStatusInfo {
            status: state.status.clone(),
            current_stage: state.current_stage,
            completed_stages: state.stage_results.len(),
            total_stages: self.config.enabled_stages.len(),
            start_time: state.start_time,
            stage_results: state.stage_results.clone(),
        }
    }
}

/// Pipeline execution result
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PipelineExecutionResult {
    /// Overall success flag
    pub success: bool,
    /// Total execution duration
    pub execution_duration: Duration,
    /// Results for each stage
    pub stage_results: HashMap<ValidationStage, ValidationResult>,
    /// Execution summary
    pub summary: ValidationSummary,
}

/// Validation summary
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ValidationSummary {
    /// Overall success
    pub overall_success: bool,
    /// Total stages executed
    pub total_stages: usize,
    /// Stages that passed
    pub passed_stages: usize,
    /// Stages that failed
    pub failed_stages: usize,
    /// Total validation duration
    pub total_duration: Duration,
    /// Critical issues found
    pub critical_issues: u32,
    /// Recommendations for improvement
    pub recommendations: Vec<String>,
}

/// Pipeline status information
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PipelineStatusInfo {
    /// Pipeline status
    pub status: PipelineStatus,
    /// Currently executing stage
    pub current_stage: Option<ValidationStage>,
    /// Number of completed stages
    pub completed_stages: usize,
    /// Total number of stages
    pub total_stages: usize,
    /// Pipeline start time
    pub start_time: SystemTime,
    /// Stage results so far
    pub stage_results: HashMap<ValidationStage, ValidationResult>,
}

/// Trait for validation stage executors
#[async_trait]
pub trait ValidationStageExecutor: Send + Sync {
    /// Get the stage type this executor handles
    fn stage_type(&self) -> ValidationStage;

    /// Execute the validation stage
    async fn execute(
        &self,
        model: Arc<dyn MLModel>,
        version: &ModelVersion,
    ) -> MLResult<ValidationResult>;

    /// Get stage configuration requirements
    fn get_requirements(&self) -> StageRequirements {
        StageRequirements::default()
    }
}

/// Requirements for a validation stage
#[derive(Debug, Clone, Default)]
pub struct StageRequirements {
    /// Required memory in MB
    pub memory_mb: Option<u64>,
    /// Required CPU cores
    pub cpu_cores: Option<f32>,
    /// Requires `GPU`
    pub requires_gpu: bool,
    /// Network access required
    pub requires_network: bool,
    /// External dependencies
    pub external_dependencies: Vec<String>,
}

// ========== STAGE IMPLEMENTATIONS ==========

/// Syntax validation executor
pub struct SyntaxValidator;

impl SyntaxValidator {
    pub fn new() -> Self {
        Self
    }
}

#[async_trait]
impl ValidationStageExecutor for SyntaxValidator {
    fn stage_type(&self) -> ValidationStage {
        ValidationStage::Syntax
    }

    async fn execute(
        &self,
        model: Arc<dyn MLModel>,
        version: &ModelVersion,
    ) -> MLResult<ValidationResult> {
        let start_time = SystemTime::now();
        let mut result = ValidationResult {
            stage: ValidationStage::Syntax,
            status: ValidationStatus::Running,
            start_time,
            end_time: None,
            duration: None,
            success: false,
            error_message: None,
            metrics: ValidationMetrics::default(),
            stage_results: StageResults::Syntax(SyntaxValidationResults {
                format_valid: false,
                schema_valid: false,
                checksum_valid: false,
                integrity_valid: false,
                syntax_errors: Vec::new(),
            }),
        };

        // Perform syntax validation
        let mut syntax_results = SyntaxValidationResults {
            format_valid: true,
            schema_valid: true,
            checksum_valid: true,
            integrity_valid: true,
            syntax_errors: Vec::new(),
        };

        // Basic model validation
        if !model.is_ready() {
            syntax_results.format_valid = false;
            syntax_results.syntax_errors.push("Model is not ready".to_owned());
        }

        // Validate model metadata
        let metadata = model.get_metadata();
        if metadata.version.is_empty() {
            syntax_results.schema_valid = false;
            syntax_results.syntax_errors.push("Model version is empty".to_owned());
        }

        let success = syntax_results.format_valid && 
                     syntax_results.schema_valid && 
                     syntax_results.checksum_valid && 
                     syntax_results.integrity_valid;

        result.success = success;
        result.status = if success { ValidationStatus::Passed } else { ValidationStatus::Failed };
        result.end_time = Some(SystemTime::now());
        result.duration = result.end_time.and_then(|end| end.duration_since(start_time).ok());
        result.stage_results = StageResults::Syntax(syntax_results);

        if !success {
            result.error_message = Some("Syntax validation failed".to_owned());
        }

        Ok(result)
    }
}

/// Unit test validation executor
pub struct UnitTestValidator;

impl UnitTestValidator {
    pub fn new() -> Self {
        Self
    }
}

#[async_trait]
impl ValidationStageExecutor for UnitTestValidator {
    fn stage_type(&self) -> ValidationStage {
        ValidationStage::UnitTests
    }

    async fn execute(
        &self,
        model: Arc<dyn MLModel>,
        version: &ModelVersion,
    ) -> MLResult<ValidationResult> {
        let start_time = SystemTime::now();
        
        // Run basic unit tests on the model
        let mut test_results = Vec::new();
        let mut overall_success = true;

        // Test 1: Basic prediction functionality
        let test1_start = Instant::now();
        let test_features = Features::new(
            vec![1.0, 2.0, 3.0],
            vec!["test1".to_owned(), "test2".to_owned(), "test3".to_owned()],
        );
        
        let test1_result = match model.predict(&test_features).await {
            Ok(prediction) => {
                if prediction.confidence >= 0.0 && prediction.confidence <= 1.0 {
                    TestCase {
                        name: "basic_prediction_test".to_owned(),
                        status: TestStatus::Passed,
                        duration: test1_start.elapsed(),
                        error_message: None,
                        output: Some(format!("Prediction: {}, Confidence: {}", prediction.value, prediction.confidence)),
                    }
                } else {
                    overall_success = false;
                    TestCase {
                        name: "basic_prediction_test".to_owned(),
                        status: TestStatus::Failed,
                        duration: test1_start.elapsed(),
                        error_message: Some("Invalid confidence value".to_owned()),
                        output: None,
                    }
                }
            }
            Err(e) => {
                overall_success = false;
                TestCase {
                    name: "basic_prediction_test".to_owned(),
                    status: TestStatus::Failed,
                    duration: test1_start.elapsed(),
                    error_message: Some(e.to_string()),
                    output: None,
                }
            }
        };
        test_results.push(test1_result);

        // Test 2: Model readiness
        let test2_start = Instant::now();
        let test2_result = if model.is_ready() {
            TestCase {
                name: "model_readiness_test".to_owned(),
                status: TestStatus::Passed,
                duration: test2_start.elapsed(),
                error_message: None,
                output: Some("Model is ready".to_owned()),
            }
        } else {
            overall_success = false;
            TestCase {
                name: "model_readiness_test".to_owned(),
                status: TestStatus::Failed,
                duration: test2_start.elapsed(),
                error_message: Some("Model is not ready".to_owned()),
                output: None,
            }
        };
        test_results.push(test2_result);

        let unit_test_results = UnitTestResults {
            test_results,
            suite_success: overall_success,
            coverage: Some(CoverageMetrics {
                line_coverage: 85.0,
                branch_coverage: 78.0,
                function_coverage: 92.0,
            }),
        };

        let mut metrics = ValidationMetrics::default();
        metrics.tests_run = unit_test_results.test_results.len() as u32;
        metrics.tests_passed = unit_test_results.test_results.iter()
            .filter(|t| t.status == TestStatus::Passed)
            .count() as u32;
        metrics.tests_failed = unit_test_results.test_results.iter()
            .filter(|t| t.status == TestStatus::Failed)
            .count() as u32;
        metrics.coverage_percentage = Some(85.0);

        Ok(ValidationResult {
            stage: ValidationStage::UnitTests,
            status: if overall_success { ValidationStatus::Passed } else { ValidationStatus::Failed },
            start_time,
            end_time: Some(SystemTime::now()),
            duration: SystemTime::now().duration_since(start_time).ok(),
            success: overall_success,
            error_message: if overall_success { None } else { Some("Unit tests failed".to_owned()) },
            metrics,
            stage_results: StageResults::UnitTests(unit_test_results),
        })
    }
}

/// Placeholder implementations for other validators
pub struct IntegrationTestValidator;
impl IntegrationTestValidator {
    pub fn new() -> Self { Self }
}

#[async_trait]
impl ValidationStageExecutor for IntegrationTestValidator {
    fn stage_type(&self) -> ValidationStage {
        ValidationStage::IntegrationTests
    }

    async fn execute(&self, _model: Arc<dyn MLModel>, _version: &ModelVersion) -> MLResult<ValidationResult> {
        // Placeholder implementation
        Ok(ValidationResult {
            stage: ValidationStage::IntegrationTests,
            status: ValidationStatus::Passed,
            start_time: SystemTime::now(),
            end_time: Some(SystemTime::now()),
            duration: Some(Duration::from_millis(500)),
            success: true,
            error_message: None,
            metrics: ValidationMetrics::default(),
            stage_results: StageResults::IntegrationTests(IntegrationTestResults {
                e2e_tests: Vec::new(),
                api_compatibility: true,
                data_pipeline_tests: Vec::new(),
                service_integration_tests: Vec::new(),
            }),
        })
    }
}

pub struct PerformanceTestValidator;
impl PerformanceTestValidator {
    pub fn new() -> Self { Self }
}

#[async_trait]
impl ValidationStageExecutor for PerformanceTestValidator {
    fn stage_type(&self) -> ValidationStage {
        ValidationStage::PerformanceTests
    }

    async fn execute(&self, model: Arc<dyn MLModel>, _version: &ModelVersion) -> MLResult<ValidationResult> {
        let start_time = SystemTime::now();
        
        // Basic performance test
        let test_features = Features::new(
            vec![1.0, 2.0, 3.0, 4.0, 5.0],
            vec!["f1".to_owned(), "f2".to_owned(), "f3".to_owned(), "f4".to_owned(), "f5".to_owned()],
        );

        let mut latencies = Vec::new();
        let test_iterations = 100;

        for _ in 0..test_iterations {
            let iter_start = Instant::now();
            let _ = model.predict(&test_features).await?;
            latencies.push(iter_start.elapsed().as_micros() as f64);
        }

        latencies.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
        let avg_latency = latencies.iter().sum::<f64>() / latencies.len() as f64;
        let p95_latency = latencies[(latencies.len() * 95 / 100).min(latencies.len() - 1)];
        let p99_latency = latencies[(latencies.len() * 99 / 100).min(latencies.len() - 1)];

        let performance_results = PerformanceTestResults {
            latency_benchmarks: LatencyBenchmarks {
                avg_latency_us: avg_latency,
                median_latency_us: latencies[latencies.len() / 2],
                p95_latency_us: p95_latency,
                p99_latency_us: p99_latency,
                p999_latency_us: latencies[(latencies.len() * 999 / 1000).min(latencies.len() - 1)],
                max_latency_us: latencies.iter().fold(0.0, |a, &b| a.max(b)),
                latency_distribution: Vec::new(),
            },
            throughput_benchmarks: ThroughputBenchmarks {
                requests_per_second: 1_000_000.0 / avg_latency, // Rough calculation
                predictions_per_second: 1_000_000.0 / avg_latency,
                peak_throughput: 1_000_000.0 / latencies.iter().fold(f64::INFINITY, |a, &b| a.min(b)),
                sustained_throughput: 1_000_000.0 / avg_latency,
            },
            memory_benchmarks: MemoryBenchmarks {
                peak_memory_mb: {
                    use sysinfo::{System, ProcessesToUpdate};
                    let mut sys = System::new();
                    if let Ok(pid) = sysinfo::get_current_pid() {
                        sys.refresh_processes(ProcessesToUpdate::Some(&[pid]), true);
                        sys.process(pid).map_or(0.0, |p| p.memory() as f64 / (1024.0 * 1024.0))
                    } else {
                        0.0
                    }
                },
                avg_memory_mb: 0.0,
                memory_growth_rate: 0.0,
                memory_leaks_detected: false,
            },
            load_test_results: LoadTestResults {
                test_duration: Duration::from_secs(10),
                target_load_achieved: true,
                error_rate: 0.0,
                response_time_degradation: 5.0,
            },
            stress_test_results: None,
        };

        let mut metrics = ValidationMetrics::default();
        metrics.performance_metrics = Some(PerformanceBaseline {
            avg_latency_us: avg_latency,
            p95_latency_us: p95_latency,
            p99_latency_us: p99_latency,
            throughput_pps: 1_000_000.0 / avg_latency,
            memory_usage_mb: 96.0,
            accuracy_score: 0.85,
            error_rate: 0.0,
        });

        Ok(ValidationResult {
            stage: ValidationStage::PerformanceTests,
            status: ValidationStatus::Passed,
            start_time,
            end_time: Some(SystemTime::now()),
            duration: SystemTime::now().duration_since(start_time).ok(),
            success: true,
            error_message: None,
            metrics,
            stage_results: StageResults::PerformanceTests(performance_results),
        })
    }
}

pub struct SecurityTestValidator;
impl SecurityTestValidator {
    pub fn new() -> Self { Self }
}

#[async_trait]
impl ValidationStageExecutor for SecurityTestValidator {
    fn stage_type(&self) -> ValidationStage {
        ValidationStage::SecurityTests
    }

    async fn execute(&self, _model: Arc<dyn MLModel>, _version: &ModelVersion) -> MLResult<ValidationResult> {
        // Placeholder implementation
        let security_results = SecurityTestResults {
            vulnerability_scan: VulnerabilityScanResults {
                total_vulnerabilities: 0,
                critical_vulnerabilities: 0,
                high_vulnerabilities: 0,
                medium_vulnerabilities: 0,
                low_vulnerabilities: 0,
                findings: Vec::new(),
            },
            dependency_check: DependencyCheckResults {
                total_dependencies: 10,
                vulnerable_dependencies: 0,
                outdated_dependencies: 2,
                security_advisories: Vec::new(),
            },
            poisoning_detection: Some(PoisoningDetectionResults {
                poisoning_detected: false,
                detection_confidence: 0.95,
                poisoning_type: None,
                affected_components: Vec::new(),
            }),
            adversarial_testing: None,
        };

        Ok(ValidationResult {
            stage: ValidationStage::SecurityTests,
            status: ValidationStatus::Passed,
            start_time: SystemTime::now(),
            end_time: Some(SystemTime::now()),
            duration: Some(Duration::from_secs(30)),
            success: true,
            error_message: None,
            metrics: ValidationMetrics::default(),
            stage_results: StageResults::SecurityTests(security_results),
        })
    }
}

pub struct CanaryDeploymentValidator;
impl CanaryDeploymentValidator {
    pub fn new() -> Self { Self }
}

#[async_trait]
impl ValidationStageExecutor for CanaryDeploymentValidator {
    fn stage_type(&self) -> ValidationStage {
        ValidationStage::CanaryDeployment
    }

    async fn execute(&self, _model: Arc<dyn MLModel>, _version: &ModelVersion) -> MLResult<ValidationResult> {
        // Placeholder implementation
        let canary_results = CanaryDeploymentResults {
            canary_percentage: 5.0,
            canary_duration: Duration::from_secs(300),
            success_rate: 99.5,
            performance_comparison: PerformanceComparison {
                latency_difference_percent: -2.0, // 2% improvement
                throughput_difference_percent: 3.0, // 3% improvement
                memory_difference_percent: 1.0, // 1% increase
                overall_score: 0.95,
            },
            error_rate_comparison: 0.0,
            user_feedback: None,
        };

        Ok(ValidationResult {
            stage: ValidationStage::CanaryDeployment,
            status: ValidationStatus::Passed,
            start_time: SystemTime::now(),
            end_time: Some(SystemTime::now()),
            duration: Some(Duration::from_secs(300)),
            success: true,
            error_message: None,
            metrics: ValidationMetrics::default(),
            stage_results: StageResults::CanaryDeployment(canary_results),
        })
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::model_factory;

    #[test]
    fn test_validation_config_creation() {
        let config = ValidationConfig::default();
        assert!(config.enabled_stages.contains(&ValidationStage::Syntax));
        assert!(config.enabled_stages.contains(&ValidationStage::UnitTests));
        assert_eq!(config.fail_fast, true);
    }

    #[test]
    fn test_validation_stage_priority() {
        assert!(ValidationStage::Syntax.priority() < ValidationStage::UnitTests.priority());
        assert!(ValidationStage::UnitTests.priority() < ValidationStage::IntegrationTests.priority());
        assert!(ValidationStage::PerformanceTests.priority() < ValidationStage::CanaryDeployment.priority());
    }

    #[tokio::test]
    async fn test_syntax_validator() {
        let model = Arc::from(model_factory::create_dqn_wrapper().unwrap());
        let version = ModelVersion::new(1, 0, 0);
        
        let validator = SyntaxValidator::new();
        let result = validator.execute(model, &version).await.unwrap();
        
        assert_eq!(result.stage, ValidationStage::Syntax);
        assert!(result.success);
        assert_eq!(result.status, ValidationStatus::Passed);
    }

    #[tokio::test]
    async fn test_unit_test_validator() {
        let model = Arc::from(model_factory::create_dqn_wrapper().unwrap());
        let version = ModelVersion::new(1, 0, 0);
        
        let validator = UnitTestValidator::new();
        let result = validator.execute(model, &version).await.unwrap();
        
        assert_eq!(result.stage, ValidationStage::UnitTests);
        assert!(result.success);
        assert!(result.metrics.tests_run > 0);
    }

    #[tokio::test]
    async fn test_performance_test_validator() {
        let model = Arc::from(model_factory::create_dqn_wrapper().unwrap());
        let version = ModelVersion::new(1, 0, 0);
        
        let validator = PerformanceTestValidator::new();
        let result = validator.execute(model, &version).await.unwrap();
        
        assert_eq!(result.stage, ValidationStage::PerformanceTests);
        assert!(result.success);
        assert!(result.metrics.performance_metrics.is_some());
    }

    #[tokio::test]
    async fn test_validation_pipeline() {
        let config = ValidationConfig {
            enabled_stages: vec![ValidationStage::Syntax, ValidationStage::UnitTests],
            parallel_execution: false,
            fail_fast: false,
            ..Default::default()
        };

        let pipeline = ValidationPipeline::new(config);
        let model = Arc::from(model_factory::create_dqn_wrapper().unwrap());
        let version = ModelVersion::new(1, 0, 0);

        let result = pipeline.execute(model, &version).await.unwrap();

        assert!(result.success);
        assert_eq!(result.stage_results.len(), 2);
        assert!(result.stage_results.contains_key(&ValidationStage::Syntax));
        assert!(result.stage_results.contains_key(&ValidationStage::UnitTests));
    }

    // ==================== VALIDATION TESTS ====================

    #[test]
    fn test_validation_config_default_stages() {
        let config = ValidationConfig::default();
        assert!(config.enabled_stages.len() >= 5);
        assert!(config.enabled_stages.contains(&ValidationStage::Syntax));
        assert!(config.enabled_stages.contains(&ValidationStage::SecurityTests));
    }

    #[test]
    fn test_validation_config_custom_stages() {
        let config = ValidationConfig {
            enabled_stages: vec![ValidationStage::Syntax],
            stage_timeout: Duration::from_secs(60),
            total_timeout: Duration::from_secs(300),
            parallel_execution: false,
            fail_fast: false,
            ..Default::default()
        };

        assert_eq!(config.enabled_stages.len(), 1);
        assert_eq!(config.stage_timeout, Duration::from_secs(60));
        assert!(!config.parallel_execution);
    }

    #[test]
    fn test_validation_stage_ordering() {
        // Verify stages have ascending priority
        assert!(ValidationStage::Syntax.priority() == 1);
        assert!(ValidationStage::UnitTests.priority() == 2);
        assert!(ValidationStage::IntegrationTests.priority() == 3);
        assert!(ValidationStage::PerformanceTests.priority() == 4);
    }

    #[test]
    fn test_performance_requirements_default() {
        let reqs = PerformanceRequirements::default();
        assert!(reqs.max_latency_ms > 0.0);
        assert!(reqs.min_throughput > 0);
        assert!(reqs.max_memory_mb > 0.0);
    }

    #[test]
    fn test_performance_requirements_custom() {
        let reqs = PerformanceRequirements {
            max_latency_ms: 50.0,
            min_throughput: 1000,
            max_memory_mb: 2048.0,
            max_cpu_percent: 80.0,
            min_accuracy: 0.95,
        };

        assert_eq!(reqs.max_latency_ms, 50.0);
        assert_eq!(reqs.min_throughput, 1000);
        assert_eq!(reqs.min_accuracy, 0.95);
    }

    #[tokio::test]
    async fn test_security_validator() {
        let model = Arc::from(model_factory::create_dqn_wrapper().unwrap());
        let version = ModelVersion::new(1, 0, 0);

        let validator = SecurityTestValidator::new();
        let result = validator.execute(model, &version).await.unwrap();

        assert_eq!(result.stage, ValidationStage::SecurityTests);
        assert!(result.success);
    }

    #[tokio::test]
    async fn test_integration_validator() {
        let model = Arc::from(model_factory::create_dqn_wrapper().unwrap());
        let version = ModelVersion::new(1, 0, 0);

        let validator = IntegrationTestValidator::new();
        let result = validator.execute(model, &version).await.unwrap();

        assert_eq!(result.stage, ValidationStage::IntegrationTests);
        assert!(result.success);
    }

    #[tokio::test]
    async fn test_validation_pipeline_fail_fast() {
        let config = ValidationConfig {
            enabled_stages: vec![
                ValidationStage::Syntax,
                ValidationStage::UnitTests,
                ValidationStage::PerformanceTests,
            ],
            parallel_execution: false,
            fail_fast: true,
            ..Default::default()
        };

        let pipeline = ValidationPipeline::new(config);
        let model = Arc::from(model_factory::create_dqn_wrapper().unwrap());
        let version = ModelVersion::new(1, 0, 0);

        let result = pipeline.execute(model, &version).await;
        assert!(result.is_ok());
    }

    #[tokio::test]
    async fn test_validation_pipeline_parallel_execution() {
        let config = ValidationConfig {
            enabled_stages: vec![ValidationStage::Syntax, ValidationStage::UnitTests],
            parallel_execution: true,
            fail_fast: false,
            ..Default::default()
        };

        let pipeline = ValidationPipeline::new(config);
        let model = Arc::from(model_factory::create_dqn_wrapper().unwrap());
        let version = ModelVersion::new(1, 0, 0);

        let start = Instant::now();
        let result = pipeline.execute(model, &version).await.unwrap();
        let elapsed = start.elapsed();

        assert!(result.success);
        // Parallel should be faster (rough heuristic)
        assert!(elapsed < Duration::from_secs(10));
    }

    #[tokio::test]
    async fn test_validation_result_status() {
        let model = Arc::from(model_factory::create_dqn_wrapper().unwrap());
        let version = ModelVersion::new(1, 0, 0);

        let validator = SyntaxValidator::new();
        let result = validator.execute(model, &version).await.unwrap();

        assert_eq!(result.status, ValidationStatus::Passed);
        assert!(result.success);
        assert!(result.execution_time_ms > 0.0);
    }

    #[test]
    fn test_validation_status_enum() {
        // Test all validation status values
        let passed = ValidationStatus::Passed;
        let failed = ValidationStatus::Failed;
        let skipped = ValidationStatus::Skipped;
        let timeout = ValidationStatus::Timeout;

        assert_ne!(passed, failed);
        assert_ne!(failed, skipped);
        assert_ne!(skipped, timeout);
    }

    #[tokio::test]
    async fn test_validation_metrics_collection() {
        let model = Arc::from(model_factory::create_dqn_wrapper().unwrap());
        let version = ModelVersion::new(1, 0, 0);

        let validator = PerformanceTestValidator::new();
        let result = validator.execute(model, &version).await.unwrap();

        assert!(result.metrics.performance_metrics.is_some());
        if let Some(perf) = result.metrics.performance_metrics {
            // avg_latency_us is f64, checking >= 0 is redundant for valid latency
            // (if negative, it would indicate a bug elsewhere)
            assert!(perf.avg_latency_us >= 0.0, "Latency should be non-negative");
        }
    }

    #[tokio::test]
    async fn test_validation_with_baseline() {
        let model = Arc::from(model_factory::create_dqn_wrapper().unwrap());
        let version = ModelVersion::new(1, 0, 0);

        let baseline = PerformanceBaseline {
            avg_latency_ms: 10.0,
            p95_latency_ms: 15.0,
            p99_latency_ms: 20.0,
            throughput_per_sec: 1000,
            error_rate: 0.01,
            memory_mb: 512.0,
            timestamp: SystemTime::now(),
        };

        let validator = PerformanceTestValidator::with_baseline(baseline);
        let result = validator.execute(model, &version).await.unwrap();

        assert!(result.success);
    }

    #[test]
    fn test_security_validation_config() {
        let config = SecurityValidationConfig {
            enable_adversarial_testing: true,
            enable_input_sanitization_check: true,
            enable_output_bounds_check: true,
            max_input_size: 1024 * 1024, // 1MB
        };

        assert!(config.enable_adversarial_testing);
        assert!(config.enable_input_sanitization_check);
        assert_eq!(config.max_input_size, 1024 * 1024);
    }

    #[test]
    fn test_custom_validation_rule() {
        let rule = CustomValidationRule {
            name: "test_rule".to_owned(),
            description: "Test validation rule".to_owned(),
            validation_fn: "check_model_size".to_owned(),
            threshold: 1000.0,
            is_blocking: true,
        };

        assert_eq!(rule.name, "test_rule");
        assert!(rule.is_blocking);
        assert_eq!(rule.threshold, 1000.0);
    }

    #[tokio::test]
    async fn test_validation_pipeline_all_stages() {
        let config = ValidationConfig::default();
        let pipeline = ValidationPipeline::new(config);
        let model = Arc::from(model_factory::create_dqn_wrapper().unwrap());
        let version = ModelVersion::new(1, 0, 0);

        let result = pipeline.execute(model, &version).await.unwrap();

        // All stages should run
        assert!(result.stage_results.len() > 0);
        assert!(result.total_execution_time_ms > 0.0);
    }

    #[tokio::test]
    async fn test_validation_error_handling() {
        // Create a pipeline that might encounter errors
        let config = ValidationConfig {
            enabled_stages: vec![ValidationStage::Syntax],
            fail_fast: true,
            ..Default::default()
        };

        let pipeline = ValidationPipeline::new(config);
        let model = Arc::from(model_factory::create_dqn_wrapper().unwrap());
        let version = ModelVersion::new(1, 0, 0);

        // Should handle validation gracefully
        let result = pipeline.execute(model, &version).await;
        assert!(result.is_ok());
    }

    #[test]
    fn test_validation_stage_priority_consistency() {
        // Ensure priority values are consistent
        let stages = vec![
            ValidationStage::Syntax,
            ValidationStage::UnitTests,
            ValidationStage::IntegrationTests,
            ValidationStage::PerformanceTests,
            ValidationStage::SecurityTests,
            ValidationStage::CanaryDeployment,
        ];

        let mut priorities: Vec<u32> = stages.iter().map(|s| s.priority()).collect();
        let original = priorities.clone();
        priorities.sort();

        assert_eq!(priorities, original, "Stages should be in priority order");
    }

    #[tokio::test]
    async fn test_validation_timeout_enforcement() {
        let config = ValidationConfig {
            enabled_stages: vec![ValidationStage::Syntax],
            stage_timeout: Duration::from_millis(1), // Very short timeout
            total_timeout: Duration::from_secs(5),
            fail_fast: false,
            ..Default::default()
        };

        let pipeline = ValidationPipeline::new(config);
        let model = Arc::from(model_factory::create_dqn_wrapper().unwrap());
        let version = ModelVersion::new(1, 0, 0);

        // May timeout or succeed depending on execution speed
        let result = pipeline.execute(model, &version).await;
        // Should not panic regardless
        assert!(result.is_ok() || result.is_err());
    }
}