foxhunt/testing/vault-integration/main.rs

//! Main test runner for Vault E2E integration tests
//!
//! This binary provides a comprehensive test runner for all Vault integration scenarios:
//! - Command-line interface for selective test execution
//! - Test result reporting and analysis
//! - Performance metrics collection
//! - HTML and JSON report generation
//! - CI/CD integration support

use anyhow::{Context, Result};
use clap::{Arg, Command};
use std::collections::HashMap;
use std::path::Path;
use std::sync::{Arc, RwLock};
use std::time::{Duration, Instant};
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use tracing::{info, warn, Level};
use tracing_subscriber;

// Type definitions for Vault E2E testing

/// Configuration for Vault E2E tests
#[derive(Debug, Clone)]
pub struct VaultTestConfig {
    /// Vault server address for testing
    pub vault_addr: String,
    /// Vault root token for test setup
    pub vault_token: String,
    /// Test timeout for individual tests
    pub test_timeout: Duration,
    /// Performance test iterations
    pub perf_iterations: u32,
    /// Docker Compose project name
    pub compose_project: String,
    /// Certificate cache directory for tests
    pub cert_cache_dir: String,
    /// Whether to cleanup resources after tests
    pub cleanup_after_tests: bool,
}

impl Default for VaultTestConfig {
    fn default() -> Self {
        Self {
            vault_addr: "http://localhost:8200".to_string(),
            vault_token: "vault-root-token".to_string(),
            test_timeout: Duration::from_secs(30),
            perf_iterations: 1000,
            compose_project: "foxhunt-vault-e2e".to_string(),
            cert_cache_dir: "/tmp/foxhunt-vault-test-certs".to_string(),
            cleanup_after_tests: true,
        }
    }
}

/// Performance metrics for test validation
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct PerformanceMetrics {
    /// Certificate generation time from Vault
    pub cert_generation_time: Option<Duration>,
    /// Certificate cache lookup time
    pub cache_lookup_time: Option<Duration>,
    /// TLS handshake time with Vault certificates
    pub tls_handshake_time: Option<Duration>,
    /// Memory usage during certificate operations
    pub memory_usage_mb: Option<f64>,
    /// CPU usage during background rotation
    pub cpu_usage_percent: Option<f64>,
    /// Vault API call count
    pub vault_api_calls: u64,
    /// Cache hit rate percentage
    pub cache_hit_rate: Option<f64>,
}

/// Results from Vault E2E test execution
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct VaultTestResults {
    /// Test execution metadata
    pub test_start_time: DateTime<Utc>,
    pub test_end_time: Option<DateTime<Utc>>,
    pub total_duration: Option<Duration>,

    /// Test execution results
    pub tests_passed: u32,
    pub tests_failed: u32,
    pub tests_skipped: u32,

    /// Performance metrics collected
    pub performance_metrics: PerformanceMetrics,

    /// Test execution details
    pub test_details: HashMap<String, TestCategoryResults>,

    /// Errors encountered during testing
    pub errors: Vec<String>,

    /// Warnings generated during testing
    pub warnings: Vec<String>,
}

/// Results from a specific test category
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct TestCategoryResults {
    pub passed: u32,
    pub failed: u32,
    pub skipped: u32,
    pub duration: Option<Duration>,
    pub details: Vec<String>,
}

mod docker_compose;
mod vault_connectivity_tests;
mod certificate_lifecycle_tests;
mod service_integration_tests;
mod failure_scenario_tests;
mod performance_impact_tests;

use docker_compose::DockerEnvironment;

/// Configuration for test execution
#[derive(Debug, Clone)]
pub struct TestExecutionConfig {
    /// Which test categories to run
    pub test_categories: Vec<String>,
    /// Whether to skip Docker cleanup
    pub skip_cleanup: bool,
    /// Output directory for reports
    pub output_dir: String,
    /// Test timeout
    pub timeout: std::time::Duration,
    /// Verbose logging
    pub verbose: bool,
    /// Run in CI mode (stricter validation)
    pub ci_mode: bool,
}

impl Default for TestExecutionConfig {
    fn default() -> Self {
        Self {
            test_categories: vec!["all".to_string()],
            skip_cleanup: false,
            output_dir: "test-results".to_string(),
            timeout: std::time::Duration::from_secs(600), // 10 minutes
            verbose: false,
            ci_mode: false,
        }
    }
}

/// Main entry point for Vault E2E tests
#[tokio::main]
async fn main() -> Result<()> {
    let matches = Command::new("Vault E2E Integration Tests")
        .version("1.0")
        .about("Comprehensive end-to-end tests for HashiCorp Vault integration")
        .arg(
            Arg::new("categories")
                .short('c')
                .long("categories")
                .value_name("CATEGORIES")
                .help("Test categories to run (comma-separated)")
                .default_value("all")
        )
        .arg(
            Arg::new("skip-cleanup")
                .long("skip-cleanup")
                .help("Skip Docker cleanup after tests")
                .action(clap::ArgAction::SetTrue)
        )
        .arg(
            Arg::new("output")
                .short('o')
                .long("output")
                .value_name("DIR")
                .help("Output directory for test reports")
                .default_value("test-results")
        )
        .arg(
            Arg::new("timeout")
                .short('t')
                .long("timeout")
                .value_name("SECONDS")
                .help("Test timeout in seconds")
                .default_value("600")
        )
        .arg(
            Arg::new("verbose")
                .short('v')
                .long("verbose")
                .help("Enable verbose logging")
                .action(clap::ArgAction::SetTrue)
        )
        .arg(
            Arg::new("ci")
                .long("ci")
                .help("Run in CI mode with stricter validation")
                .action(clap::ArgAction::SetTrue)
        )
        .get_matches();

    // Initialize logging
    let log_level = if matches.get_flag("verbose") {
        Level::DEBUG
    } else {
        Level::INFO
    };
    
    tracing_subscriber::fmt()
        .with_max_level(log_level)
        .with_target(false)
        .init();

    // Parse configuration
    let categories: Vec<String> = matches.get_one::<String>("categories")
        .unwrap()
        .split(',')
        .map(|s| s.trim().to_string())
        .collect();

    let timeout_seconds: u64 = matches.get_one::<String>("timeout")
        .unwrap()
        .parse()
        .context("Invalid timeout value")?;

    let config = TestExecutionConfig {
        test_categories: categories,
        skip_cleanup: matches.get_flag("skip-cleanup"),
        output_dir: matches.get_one::<String>("output").unwrap().to_string(),
        timeout: std::time::Duration::from_secs(timeout_seconds),
        verbose: matches.get_flag("verbose"),
        ci_mode: matches.get_flag("ci"),
    };

    info!("Starting Vault E2E integration tests");
    info!("Test categories: {:?}", config.test_categories);
    info!("Output directory: {}", config.output_dir);
    info!("Timeout: {:?}", config.timeout);

    // Create output directory
    std::fs::create_dir_all(&config.output_dir)
        .with_context(|| format!("Failed to create output directory: {}", config.output_dir))?;

    // Run tests
    let test_start = Instant::now();
    let result = run_vault_e2e_tests(config).await;
    let total_duration = test_start.elapsed();

    match result {
        Ok(test_results) => {
            info!("Vault E2E tests completed successfully in {:?}", total_duration);
            
            // Print final report
            println!("{}", test_results.generate_report());
            
            // Generate detailed reports
            generate_test_reports(&test_results, &config.output_dir).await?;
            
            // Determine exit code based on test results
            if test_results.failure_count() > 0 {
                if config.ci_mode {
                    std::process::exit(1);
                } else {
                    warn!("Some tests failed, but exiting with success in non-CI mode");
                }
            }
        }
        Err(e) => {
            eprintln!("Vault E2E tests failed: {}", e);
            std::process::exit(1);
        }
    }

    Ok(())
}

/// Run comprehensive Vault E2E tests
async fn run_vault_e2e_tests(config: TestExecutionConfig) -> Result<VaultTestResults> {
    let vault_config = VaultTestConfig {
        cleanup_after_tests: !config.skip_cleanup,
        test_timeout: config.timeout,
        ..Default::default()
    };

    let test_suite = VaultE2ETestSuite::new(vault_config);
    
    // Run tests with timeout
    let test_result = tokio::time::timeout(
        config.timeout,
        test_suite.run_selected_tests(&config.test_categories)
    ).await;

    match test_result {
        Ok(results) => results,
        Err(_) => Err(anyhow::anyhow!("Tests timed out after {:?}", config.timeout)),
    }
}

/// Enhanced test suite with selective test execution
pub struct VaultE2ETestSuite {
    config: VaultTestConfig,
}

impl VaultE2ETestSuite {
    pub fn new(config: VaultTestConfig) -> Self {
        Self { config }
    }

    /// Run selected test categories
    pub async fn run_selected_tests(&self, categories: &[String]) -> Result<VaultTestResults> {
        use std::sync::Arc;
        use tokio::sync::RwLock;
        
        let results = Arc::new(RwLock::new(VaultTestResults::new()));
        
        info!("Running selected test categories: {:?}", categories);

        // Add test metadata
        {
            let mut test_results = results.write().await;
            test_results.add_metadata("test_started".to_string(), chrono::Utc::now().to_rfc3339());
            test_results.add_metadata("vault_addr".to_string(), self.config.vault_addr.clone());
            test_results.add_metadata("categories".to_string(), categories.join(","));
        }

        let should_run_all = categories.contains(&"all".to_string());

        // Test 1: Docker environment setup (always required)
        if should_run_all || categories.contains(&"docker".to_string()) || categories.contains(&"setup".to_string()) {
            self.run_docker_setup_tests(&results).await?;
        }

        // Test 2: Vault connectivity
        if should_run_all || categories.contains(&"connectivity".to_string()) || categories.contains(&"basic".to_string()) {
            self.run_connectivity_tests(&results).await?;
        }

        // Test 3: Certificate lifecycle
        if should_run_all || categories.contains(&"certificates".to_string()) || categories.contains(&"lifecycle".to_string()) {
            self.run_certificate_tests(&results).await?;
        }

        // Test 4: Service integration
        if should_run_all || categories.contains(&"services".to_string()) || categories.contains(&"integration".to_string()) {
            self.run_service_integration_tests(&results).await?;
        }

        // Test 5: Failure scenarios
        if should_run_all || categories.contains(&"failures".to_string()) || categories.contains(&"resilience".to_string()) {
            self.run_failure_scenario_tests(&results).await?;
        }

        // Test 6: Performance validation
        if should_run_all || categories.contains(&"performance".to_string()) || categories.contains(&"hft".to_string()) {
            self.run_performance_tests(&results).await?;
        }

        // Test 7: Cleanup
        if self.config.cleanup_after_tests && (should_run_all || categories.contains(&"cleanup".to_string())) {
            self.run_cleanup_tests(&results).await?;
        }

        let final_results = results.read().await;
        info!("Test execution completed: {}/{} tests passed", 
              final_results.success_count(), final_results.total_count());

        Ok(final_results.clone())
    }

    // Individual test category methods (similar to the main implementation)
    async fn run_docker_setup_tests(&self, results: &Arc<RwLock<VaultTestResults>>) -> Result<()> {
        info!("Running Docker environment setup tests");
        
        let test_start = Instant::now();
        match DockerEnvironment::new(&self.config).await {
            Ok(mut env) => {
                if let Err(e) = env.start_all_services().await {
                    let mut test_results = results.write().await;
                    test_results.add_failure("docker_environment_startup", e.to_string());
                    return Err(e);
                }
                
                let mut test_results = results.write().await;
                test_results.add_success("docker_environment_startup", test_start.elapsed());
                Ok(())
            }
            Err(e) => {
                let mut test_results = results.write().await;
                test_results.add_failure("docker_environment_setup", e.to_string());
                Err(e)
            }
        }
    }

    async fn run_connectivity_tests(&self, results: &Arc<RwLock<VaultTestResults>>) -> Result<()> {
        info!("Running Vault connectivity tests");
        
        match vault_connectivity_tests::run_connectivity_tests(&self.config).await {
            Ok(duration) => {
                let mut test_results = results.write().await;
                test_results.add_success("vault_connectivity", duration);
                Ok(())
            }
            Err(e) => {
                let mut test_results = results.write().await;
                test_results.add_failure("vault_connectivity", e.to_string());
                Err(e)
            }
        }
    }

    async fn run_certificate_tests(&self, results: &Arc<RwLock<VaultTestResults>>) -> Result<()> {
        certificate_lifecycle_tests::run_certificate_tests(&self.config, results).await
    }

    async fn run_service_integration_tests(&self, results: &Arc<RwLock<VaultTestResults>>) -> Result<()> {
        service_integration_tests::run_service_tests(&self.config, results).await
    }

    async fn run_failure_scenario_tests(&self, results: &Arc<RwLock<VaultTestResults>>) -> Result<()> {
        failure_scenario_tests::run_failure_tests(&self.config, results).await
    }

    async fn run_performance_tests(&self, results: &Arc<RwLock<VaultTestResults>>) -> Result<()> {
        performance_impact_tests::run_performance_tests(&self.config, results).await
    }

    async fn run_cleanup_tests(&self, results: &Arc<RwLock<VaultTestResults>>) -> Result<()> {
        let test_start = Instant::now();
        if let Err(e) = DockerEnvironment::cleanup(&self.config).await {
            warn!("Docker cleanup failed: {}", e);
        }

        let mut test_results = results.write().await;
        test_results.add_success("cleanup", test_start.elapsed());
        Ok(())
    }
}

/// Generate comprehensive test reports
async fn generate_test_reports(results: &VaultTestResults, output_dir: &str) -> Result<()> {
    info!("Generating test reports in: {}", output_dir);

    // Generate JSON report
    let json_report = serde_json::to_string_pretty(results)
        .context("Failed to serialize test results to JSON")?;
    
    let json_path = format!("{}/vault_e2e_results.json", output_dir);
    tokio::fs::write(&json_path, json_report).await
        .with_context(|| format!("Failed to write JSON report: {}", json_path))?;

    // Generate text report
    let text_report = results.generate_report();
    let text_path = format!("{}/vault_e2e_results.txt", output_dir);
    tokio::fs::write(&text_path, text_report).await
        .with_context(|| format!("Failed to write text report: {}", text_path))?;

    // Generate HTML report (simplified)
    let html_report = generate_html_report(results);
    let html_path = format!("{}/vault_e2e_results.html", output_dir);
    tokio::fs::write(&html_path, html_report).await
        .with_context(|| format!("Failed to write HTML report: {}", html_path))?;

    info!("Test reports generated:");
    info!("  JSON: {}", json_path);
    info!("  Text: {}", text_path);
    info!("  HTML: {}", html_path);

    Ok(())
}

/// Generate HTML test report
fn generate_html_report(results: &VaultTestResults) -> String {
    let success_rate = results.success_rate() * 100.0;
    let status_color = if success_rate >= 95.0 { "green" } else if success_rate >= 80.0 { "orange" } else { "red" };

    format!(r#"<!DOCTYPE html>
<html>
<head>
    <title>Vault E2E Test Results</title>
    <style>
        body {{ font-family: Arial, sans-serif; margin: 40px; }}
        .header {{ background: #f5f5f5; padding: 20px; border-radius: 5px; }}
        .success {{ color: green; }}
        .failure {{ color: red; }}
        .metric {{ margin: 10px 0; }}
        .performance {{ background: #f0f8ff; padding: 15px; border-radius: 5px; margin: 20px 0; }}
        table {{ border-collapse: collapse; width: 100%; margin: 20px 0; }}
        th, td {{ border: 1px solid #ddd; padding: 8px; text-align: left; }}
        th {{ background-color: #f2f2f2; }}
    </style>
</head>
<body>
    <div class="header">
        <h1>Vault E2E Integration Test Results</h1>
        <p><strong>Success Rate:</strong> <span style="color: {status_color}">{success_rate:.1}%</span></p>
        <p><strong>Total Tests:</strong> {total}</p>
        <p><strong>Passed:</strong> <span class="success">{passed}</span></p>
        <p><strong>Failed:</strong> <span class="failure">{failed}</span></p>
    </div>

    <div class="performance">
        <h2>Performance Metrics</h2>
        <div class="metric"><strong>Certificate Generation:</strong> {cert_time}</div>
        <div class="metric"><strong>Cache Lookup:</strong> {cache_time}</div>
        <div class="metric"><strong>TLS Handshake:</strong> {tls_time}</div>
        <div class="metric"><strong>Memory Usage:</strong> {memory}</div>
        <div class="metric"><strong>Cache Hit Rate:</strong> {hit_rate}</div>
    </div>

    <h2>Test Results Details</h2>
    <table>
        <tr><th>Test Name</th><th>Status</th><th>Duration</th><th>Details</th></tr>
        {test_rows}
    </table>
</body>
</html>"#,
        status_color = status_color,
        success_rate = success_rate,
        total = results.total_count(),
        passed = results.success_count(),
        failed = results.failure_count(),
        cert_time = results.performance.cert_generation_time.map(|d| format!("{:?}", d)).unwrap_or("N/A".to_string()),
        cache_time = results.performance.cache_lookup_time.map(|d| format!("{:?}", d)).unwrap_or("N/A".to_string()),
        tls_time = results.performance.tls_handshake_time.map(|d| format!("{:?}", d)).unwrap_or("N/A".to_string()),
        memory = results.performance.memory_usage_mb.map(|m| format!("{:.1} MB", m)).unwrap_or("N/A".to_string()),
        hit_rate = results.performance.cache_hit_rate.map(|r| format!("{:.1}%", r)).unwrap_or("N/A".to_string()),
        test_rows = generate_test_table_rows(results),
    )
}

fn generate_test_table_rows(results: &VaultTestResults) -> String {
    let mut rows = String::new();
    
    for (name, duration) in &results.successes {
        rows.push_str(&format!(
            "<tr><td>{}</td><td class=\"success\">PASSED</td><td>{:?}</td><td>-</td></tr>",
            name, duration
        ));
    }
    
    for (name, error) in &results.failures {
        rows.push_str(&format!(
            "<tr><td>{}</td><td class=\"failure\">FAILED</td><td>-</td><td>{}</td></tr>",
            name, error
        ));
    }
    
    rows
}

// Re-export the types and functions from the main module
// Types are now defined directly in this file