foxhunt/testing/vault-integration/mod.rs

//! Comprehensive End-to-End Tests for HashiCorp Vault Integration
//!
//! This module provides comprehensive E2E testing for Vault integration in the Foxhunt HFT system:
//! - Real Vault server testing with PKI secrets engine
//! - Service startup dependency validation
//! - Certificate lifecycle management (generation, caching, rotation)
//! - Failure scenario testing (Vault down, network partitions, timeouts)
//! - Performance impact measurement for HFT requirements
//! - Circuit breaker behavior validation
//! - Multi-service integration testing

use anyhow::{Context, Result};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::RwLock;
use tracing::{debug, info, warn, error};

// Re-export test modules
pub mod docker_compose;
pub mod vault_connectivity_tests;
pub mod certificate_lifecycle_tests;
pub mod service_integration_tests;
pub mod failure_scenario_tests;
pub mod performance_impact_tests;

/// Configuration for Vault E2E tests
#[derive(Debug, Clone)]
pub struct VaultTestConfig {
    /// Vault server address for testing
    pub vault_addr: String,
    /// Vault root token for test setup
    pub vault_token: String,
    /// Test timeout for individual tests
    pub test_timeout: Duration,
    /// Performance test iterations
    pub perf_iterations: u32,
    /// Docker Compose project name
    pub compose_project: String,
    /// Certificate cache directory for tests
    pub cert_cache_dir: String,
    /// Whether to cleanup resources after tests
    pub cleanup_after_tests: bool,
}

impl Default for VaultTestConfig {
    fn default() -> Self {
        Self {
            vault_addr: "http://localhost:8200".to_string(),
            vault_token: "vault-root-token".to_string(),
            test_timeout: Duration::from_secs(30),
            perf_iterations: 1000,
            compose_project: "foxhunt-vault-e2e".to_string(),
            cert_cache_dir: "/tmp/foxhunt-vault-test-certs".to_string(),
            cleanup_after_tests: true,
        }
    }
}

/// Performance metrics for test validation
#[derive(Debug, Clone, Default)]
pub struct PerformanceMetrics {
    /// Certificate generation time from Vault
    pub cert_generation_time: Option<Duration>,
    /// Certificate cache lookup time
    pub cache_lookup_time: Option<Duration>,
    /// TLS handshake time with Vault certificates
    pub tls_handshake_time: Option<Duration>,
    /// Memory usage during certificate operations
    pub memory_usage_mb: Option<f64>,
    /// CPU usage during background rotation
    pub cpu_usage_percent: Option<f64>,
    /// Vault API call count
    pub vault_api_calls: u64,
    /// Cache hit rate percentage
    pub cache_hit_rate: Option<f64>,
}

impl PerformanceMetrics {
    pub fn new() -> Self {
        Self::default()
    }

    /// Validate metrics against HFT requirements
    pub fn validate_hft_requirements(&self) -> Result<()> {
        // Certificate generation should be under 100ms
        if let Some(cert_time) = self.cert_generation_time {
            if cert_time > Duration::from_millis(100) {
                return Err(anyhow::anyhow!(
                    "Certificate generation time {}ms exceeds 100ms HFT requirement",
                    cert_time.as_millis()
                ));
            }
        }

        // Cache lookup should be under 1μs
        if let Some(cache_time) = self.cache_lookup_time {
            if cache_time > Duration::from_micros(1) {
                return Err(anyhow::anyhow!(
                    "Cache lookup time {}μs exceeds 1μs HFT requirement",
                    cache_time.as_micros()
                ));
            }
        }

        // TLS handshake should be under 1ms for cached certificates
        if let Some(tls_time) = self.tls_handshake_time {
            if tls_time > Duration::from_millis(1) {
                return Err(anyhow::anyhow!(
                    "TLS handshake time {}ms exceeds 1ms HFT requirement",
                    tls_time.as_millis()
                ));
            }
        }

        // Memory usage should be under 10MB per service
        if let Some(memory) = self.memory_usage_mb {
            if memory > 10.0 {
                return Err(anyhow::anyhow!(
                    "Memory usage {:.1}MB exceeds 10MB HFT requirement",
                    memory
                ));
            }
        }

        // CPU usage should be under 5% for background tasks
        if let Some(cpu) = self.cpu_usage_percent {
            if cpu > 5.0 {
                return Err(anyhow::anyhow!(
                    "CPU usage {:.1}% exceeds 5% HFT requirement",
                    cpu
                ));
            }
        }

        // Cache hit rate should be over 80%
        if let Some(hit_rate) = self.cache_hit_rate {
            if hit_rate < 80.0 {
                return Err(anyhow::anyhow!(
                    "Cache hit rate {:.1}% is below 80% efficiency requirement",
                    hit_rate
                ));
            }
        }

        Ok(())
    }
}

/// Test results aggregation for comprehensive reporting
#[derive(Debug, Default)]
pub struct VaultTestResults {
    /// Successful tests with timing
    pub successes: Vec<(String, Duration)>,
    /// Failed tests with error messages
    pub failures: Vec<(String, String)>,
    /// Performance metrics collected
    pub performance: PerformanceMetrics,
    /// Additional metadata
    pub metadata: HashMap<String, String>,
}

impl VaultTestResults {
    pub fn new() -> Self {
        Self::default()
    }

    pub fn add_success(&mut self, test_name: &str, duration: Duration) {
        self.successes.push((test_name.to_string(), duration));
        info!("✓ Test passed: {} ({}ms)", test_name, duration.as_millis());
    }

    pub fn add_failure(&mut self, test_name: &str, error: String) {
        self.failures.push((test_name.to_string(), error.clone()));
        error!("✗ Test failed: {} - {}", test_name, error);
    }

    pub fn add_metadata(&mut self, key: &str, value: String) {
        self.metadata.insert(key.to_string(), value);
    }

    pub fn success_count(&self) -> usize {
        self.successes.len()
    }

    pub fn failure_count(&self) -> usize {
        self.failures.len()
    }

    pub fn total_count(&self) -> usize {
        self.successes.len() + self.failures.len()
    }

    pub fn success_rate(&self) -> f64 {
        if self.total_count() == 0 {
            return 0.0;
        }
        self.success_count() as f64 / self.total_count() as f64
    }

    /// Generate comprehensive test report
    pub fn generate_report(&self) -> String {
        let mut report = String::new();
        
        report.push_str("\n======= VAULT E2E TEST RESULTS =======\n");
        report.push_str(&format!("Total Tests: {}\n", self.total_count()));
        report.push_str(&format!("Successes: {}\n", self.success_count()));
        report.push_str(&format!("Failures: {}\n", self.failure_count()));
        report.push_str(&format!("Success Rate: {:.1}%\n", self.success_rate() * 100.0));

        // Performance metrics
        report.push_str("\n--- Performance Metrics ---\n");
        if let Some(cert_time) = self.performance.cert_generation_time {
            report.push_str(&format!("Certificate Generation: {}ms\n", cert_time.as_millis()));
        }
        if let Some(cache_time) = self.performance.cache_lookup_time {
            report.push_str(&format!("Cache Lookup: {}μs\n", cache_time.as_micros()));
        }
        if let Some(tls_time) = self.performance.tls_handshake_time {
            report.push_str(&format!("TLS Handshake: {}ms\n", tls_time.as_millis()));
        }
        if let Some(memory) = self.performance.memory_usage_mb {
            report.push_str(&format!("Memory Usage: {:.1}MB\n", memory));
        }
        if let Some(cpu) = self.performance.cpu_usage_percent {
            report.push_str(&format!("CPU Usage: {:.1}%\n", cpu));
        }
        if let Some(hit_rate) = self.performance.cache_hit_rate {
            report.push_str(&format!("Cache Hit Rate: {:.1}%\n", hit_rate));
        }
        report.push_str(&format!("Vault API Calls: {}\n", self.performance.vault_api_calls));

        // HFT requirements validation
        report.push_str("\n--- HFT Requirements Validation ---\n");
        match self.performance.validate_hft_requirements() {
            Ok(()) => report.push_str("✓ All HFT performance requirements met\n"),
            Err(e) => report.push_str(&format!("✗ HFT requirement violation: {}\n", e)),
        }

        // Successful tests
        if !self.successes.is_empty() {
            report.push_str("\n--- Successful Tests ---\n");
            for (name, duration) in &self.successes {
                report.push_str(&format!("✓ {} - {}ms\n", name, duration.as_millis()));
            }
        }

        // Failed tests
        if !self.failures.is_empty() {
            report.push_str("\n--- Failed Tests ---\n");
            for (name, error) in &self.failures {
                report.push_str(&format!("✗ {} - {}\n", name, error));
            }
        }

        // Metadata
        if !self.metadata.is_empty() {
            report.push_str("\n--- Test Metadata ---\n");
            for (key, value) in &self.metadata {
                report.push_str(&format!("{}: {}\n", key, value));
            }
        }

        report.push_str("======================================\n");
        report
    }
}

/// Main Vault E2E test suite coordinator
pub struct VaultE2ETestSuite {
    config: VaultTestConfig,
    results: Arc<RwLock<VaultTestResults>>,
}

impl VaultE2ETestSuite {
    /// Create new test suite with configuration
    pub fn new(config: VaultTestConfig) -> Self {
        Self {
            config,
            results: Arc::new(RwLock::new(VaultTestResults::new())),
        }
    }

    /// Run all Vault E2E tests in sequence
    pub async fn run_all_tests(&self) -> Result<VaultTestResults> {
        info!("Starting comprehensive Vault E2E test suite");
        
        // Add test metadata
        {
            let mut results = self.results.write().await;
            results.add_metadata("vault_addr".to_string(), self.config.vault_addr.clone());
            results.add_metadata("test_started".to_string(), chrono::Utc::now().to_rfc3339());
        }

        // Test 1: Docker environment setup
        self.run_docker_setup_tests().await?;

        // Test 2: Basic Vault connectivity
        self.run_connectivity_tests().await?;

        // Test 3: Certificate lifecycle management
        self.run_certificate_tests().await?;

        // Test 4: Service integration testing
        self.run_service_integration_tests().await?;

        // Test 5: Failure scenario testing
        self.run_failure_scenario_tests().await?;

        // Test 6: Performance impact validation
        self.run_performance_tests().await?;

        // Test 7: Cleanup and validation
        if self.config.cleanup_after_tests {
            self.run_cleanup_tests().await?;
        }

        let results = self.results.read().await;
        info!("Vault E2E test suite completed: {}/{} tests passed", 
              results.success_count(), results.total_count());

        Ok(results.clone())
    }

    /// Run Docker environment setup tests
    async fn run_docker_setup_tests(&self) -> Result<()> {
        info!("Running Docker environment setup tests");
        
        let test_start = Instant::now();
        match docker_compose::DockerEnvironment::new(&self.config).await {
            Ok(mut env) => {
                if let Err(e) = env.start_all_services().await {
                    let mut results = self.results.write().await;
                    results.add_failure("docker_environment_startup", e.to_string());
                    return Err(e);
                }
                
                let mut results = self.results.write().await;
                results.add_success("docker_environment_startup", test_start.elapsed());
                Ok(())
            }
            Err(e) => {
                let mut results = self.results.write().await;
                results.add_failure("docker_environment_setup", e.to_string());
                Err(e)
            }
        }
    }

    /// Run Vault connectivity tests
    async fn run_connectivity_tests(&self) -> Result<()> {
        info!("Running Vault connectivity tests");
        
        match vault_connectivity_tests::run_connectivity_tests(&self.config).await {
            Ok(duration) => {
                let mut results = self.results.write().await;
                results.add_success("vault_connectivity", duration);
                Ok(())
            }
            Err(e) => {
                let mut results = self.results.write().await;
                results.add_failure("vault_connectivity", e.to_string());
                Err(e)
            }
        }
    }

    /// Run certificate lifecycle tests
    async fn run_certificate_tests(&self) -> Result<()> {
        info!("Running certificate lifecycle tests");
        
        match certificate_lifecycle_tests::run_certificate_tests(&self.config, &self.results).await {
            Ok(()) => Ok(()),
            Err(e) => {
                let mut results = self.results.write().await;
                results.add_failure("certificate_lifecycle", e.to_string());
                Err(e)
            }
        }
    }

    /// Run service integration tests
    async fn run_service_integration_tests(&self) -> Result<()> {
        info!("Running service integration tests");
        
        match service_integration_tests::run_service_tests(&self.config, &self.results).await {
            Ok(()) => Ok(()),
            Err(e) => {
                let mut results = self.results.write().await;
                results.add_failure("service_integration", e.to_string());
                Err(e)
            }
        }
    }

    /// Run failure scenario tests
    async fn run_failure_scenario_tests(&self) -> Result<()> {
        info!("Running failure scenario tests");
        
        match failure_scenario_tests::run_failure_tests(&self.config, &self.results).await {
            Ok(()) => Ok(()),
            Err(e) => {
                let mut results = self.results.write().await;
                results.add_failure("failure_scenarios", e.to_string());
                Err(e)
            }
        }
    }

    /// Run performance impact tests
    async fn run_performance_tests(&self) -> Result<()> {
        info!("Running performance impact tests");
        
        match performance_impact_tests::run_performance_tests(&self.config, &self.results).await {
            Ok(()) => Ok(()),
            Err(e) => {
                let mut results = self.results.write().await;
                results.add_failure("performance_validation", e.to_string());
                Err(e)
            }
        }
    }

    /// Run cleanup and validation tests
    async fn run_cleanup_tests(&self) -> Result<()> {
        info!("Running cleanup tests");
        
        let test_start = Instant::now();
        // Cleanup Docker resources
        if let Err(e) = docker_compose::DockerEnvironment::cleanup(&self.config).await {
            warn!("Docker cleanup failed: {}", e);
        }

        let mut results = self.results.write().await;
        results.add_success("cleanup", test_start.elapsed());
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_vault_test_config() {
        let config = VaultTestConfig::default();
        assert_eq!(config.vault_addr, "http://localhost:8200");
        assert_eq!(config.vault_token, "vault-root-token");
        assert!(config.cleanup_after_tests);
    }

    #[test]
    fn test_performance_metrics_validation() {
        let mut metrics = PerformanceMetrics::new();
        
        // Should pass with good metrics
        metrics.cert_generation_time = Some(Duration::from_millis(50));
        metrics.cache_lookup_time = Some(Duration::from_nanos(500));
        metrics.tls_handshake_time = Some(Duration::from_millis(1));
        metrics.memory_usage_mb = Some(5.0);
        metrics.cpu_usage_percent = Some(2.0);
        metrics.cache_hit_rate = Some(90.0);
        
        assert!(metrics.validate_hft_requirements().is_ok());

        // Should fail with bad metrics
        metrics.cert_generation_time = Some(Duration::from_millis(150));
        assert!(metrics.validate_hft_requirements().is_err());
    }

    #[test]
    fn test_results_aggregation() {
        let mut results = VaultTestResults::new();
        results.add_success("test1", Duration::from_millis(10));
        results.add_failure("test2", "Test error".to_string());
        
        assert_eq!(results.success_count(), 1);
        assert_eq!(results.failure_count(), 1);
        assert_eq!(results.success_rate(), 0.5);
        
        let report = results.generate_report();
        assert!(report.contains("Total Tests: 2"));
        assert!(report.contains("Success Rate: 50.0%"));
    }
}