//! Main test runner for Vault E2E integration tests //! //! This binary provides a comprehensive test runner for all Vault integration scenarios: //! - Command-line interface for selective test execution //! - Test result reporting and analysis //! - Performance metrics collection //! - HTML and JSON report generation //! - CI/CD integration support use anyhow::{Context, Result}; use clap::{Arg, Command}; use std::collections::HashMap; use std::path::Path; use std::sync::{Arc, RwLock}; use std::time::{Duration, Instant}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use tracing::{info, warn, Level}; use tracing_subscriber; // Type definitions for Vault E2E testing /// Configuration for Vault E2E tests #[derive(Debug, Clone)] pub struct VaultTestConfig { /// Vault server address for testing pub vault_addr: String, /// Vault root token for test setup pub vault_token: String, /// Test timeout for individual tests pub test_timeout: Duration, /// Performance test iterations pub perf_iterations: u32, /// Docker Compose project name pub compose_project: String, /// Certificate cache directory for tests pub cert_cache_dir: String, /// Whether to cleanup resources after tests pub cleanup_after_tests: bool, } impl Default for VaultTestConfig { fn default() -> Self { Self { vault_addr: "http://localhost:8200".to_string(), vault_token: "vault-root-token".to_string(), test_timeout: Duration::from_secs(30), perf_iterations: 1000, compose_project: "foxhunt-vault-e2e".to_string(), cert_cache_dir: "/tmp/foxhunt-vault-test-certs".to_string(), cleanup_after_tests: true, } } } /// Performance metrics for test validation #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct PerformanceMetrics { /// Certificate generation time from Vault pub cert_generation_time: Option, /// Certificate cache lookup time pub cache_lookup_time: Option, /// TLS handshake time with Vault certificates pub tls_handshake_time: Option, /// Memory usage during certificate operations pub memory_usage_mb: Option, /// CPU usage during background rotation pub cpu_usage_percent: Option, /// Vault API call count pub vault_api_calls: u64, /// Cache hit rate percentage pub cache_hit_rate: Option, } /// Results from Vault E2E test execution #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct VaultTestResults { /// Test execution metadata pub test_start_time: DateTime, pub test_end_time: Option>, pub total_duration: Option, /// Test execution results pub tests_passed: u32, pub tests_failed: u32, pub tests_skipped: u32, /// Performance metrics collected pub performance_metrics: PerformanceMetrics, /// Test execution details pub test_details: HashMap, /// Errors encountered during testing pub errors: Vec, /// Warnings generated during testing pub warnings: Vec, } /// Results from a specific test category #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct TestCategoryResults { pub passed: u32, pub failed: u32, pub skipped: u32, pub duration: Option, pub details: Vec, } mod docker_compose; mod vault_connectivity_tests; mod certificate_lifecycle_tests; mod service_integration_tests; mod failure_scenario_tests; mod performance_impact_tests; use docker_compose::DockerEnvironment; /// Configuration for test execution #[derive(Debug, Clone)] pub struct TestExecutionConfig { /// Which test categories to run pub test_categories: Vec, /// Whether to skip Docker cleanup pub skip_cleanup: bool, /// Output directory for reports pub output_dir: String, /// Test timeout pub timeout: std::time::Duration, /// Verbose logging pub verbose: bool, /// Run in CI mode (stricter validation) pub ci_mode: bool, } impl Default for TestExecutionConfig { fn default() -> Self { Self { test_categories: vec!["all".to_string()], skip_cleanup: false, output_dir: "test-results".to_string(), timeout: std::time::Duration::from_secs(600), // 10 minutes verbose: false, ci_mode: false, } } } /// Main entry point for Vault E2E tests #[tokio::main] async fn main() -> Result<()> { let matches = Command::new("Vault E2E Integration Tests") .version("1.0") .about("Comprehensive end-to-end tests for HashiCorp Vault integration") .arg( Arg::new("categories") .short('c') .long("categories") .value_name("CATEGORIES") .help("Test categories to run (comma-separated)") .default_value("all") ) .arg( Arg::new("skip-cleanup") .long("skip-cleanup") .help("Skip Docker cleanup after tests") .action(clap::ArgAction::SetTrue) ) .arg( Arg::new("output") .short('o') .long("output") .value_name("DIR") .help("Output directory for test reports") .default_value("test-results") ) .arg( Arg::new("timeout") .short('t') .long("timeout") .value_name("SECONDS") .help("Test timeout in seconds") .default_value("600") ) .arg( Arg::new("verbose") .short('v') .long("verbose") .help("Enable verbose logging") .action(clap::ArgAction::SetTrue) ) .arg( Arg::new("ci") .long("ci") .help("Run in CI mode with stricter validation") .action(clap::ArgAction::SetTrue) ) .get_matches(); // Initialize logging let log_level = if matches.get_flag("verbose") { Level::DEBUG } else { Level::INFO }; tracing_subscriber::fmt() .with_max_level(log_level) .with_target(false) .init(); // Parse configuration let categories: Vec = matches.get_one::("categories") .unwrap() .split(',') .map(|s| s.trim().to_string()) .collect(); let timeout_seconds: u64 = matches.get_one::("timeout") .unwrap() .parse() .context("Invalid timeout value")?; let config = TestExecutionConfig { test_categories: categories, skip_cleanup: matches.get_flag("skip-cleanup"), output_dir: matches.get_one::("output").unwrap().to_string(), timeout: std::time::Duration::from_secs(timeout_seconds), verbose: matches.get_flag("verbose"), ci_mode: matches.get_flag("ci"), }; info!("Starting Vault E2E integration tests"); info!("Test categories: {:?}", config.test_categories); info!("Output directory: {}", config.output_dir); info!("Timeout: {:?}", config.timeout); // Create output directory std::fs::create_dir_all(&config.output_dir) .with_context(|| format!("Failed to create output directory: {}", config.output_dir))?; // Run tests let test_start = Instant::now(); let result = run_vault_e2e_tests(config).await; let total_duration = test_start.elapsed(); match result { Ok(test_results) => { info!("Vault E2E tests completed successfully in {:?}", total_duration); // Print final report println!("{}", test_results.generate_report()); // Generate detailed reports generate_test_reports(&test_results, &config.output_dir).await?; // Determine exit code based on test results if test_results.failure_count() > 0 { if config.ci_mode { std::process::exit(1); } else { warn!("Some tests failed, but exiting with success in non-CI mode"); } } } Err(e) => { eprintln!("Vault E2E tests failed: {}", e); std::process::exit(1); } } Ok(()) } /// Run comprehensive Vault E2E tests async fn run_vault_e2e_tests(config: TestExecutionConfig) -> Result { let vault_config = VaultTestConfig { cleanup_after_tests: !config.skip_cleanup, test_timeout: config.timeout, ..Default::default() }; let test_suite = VaultE2ETestSuite::new(vault_config); // Run tests with timeout let test_result = tokio::time::timeout( config.timeout, test_suite.run_selected_tests(&config.test_categories) ).await; match test_result { Ok(results) => results, Err(_) => Err(anyhow::anyhow!("Tests timed out after {:?}", config.timeout)), } } /// Enhanced test suite with selective test execution pub struct VaultE2ETestSuite { config: VaultTestConfig, } impl VaultE2ETestSuite { pub fn new(config: VaultTestConfig) -> Self { Self { config } } /// Run selected test categories pub async fn run_selected_tests(&self, categories: &[String]) -> Result { use std::sync::Arc; use tokio::sync::RwLock; let results = Arc::new(RwLock::new(VaultTestResults::new())); info!("Running selected test categories: {:?}", categories); // Add test metadata { let mut test_results = results.write().await; test_results.add_metadata("test_started".to_string(), chrono::Utc::now().to_rfc3339()); test_results.add_metadata("vault_addr".to_string(), self.config.vault_addr.clone()); test_results.add_metadata("categories".to_string(), categories.join(",")); } let should_run_all = categories.contains(&"all".to_string()); // Test 1: Docker environment setup (always required) if should_run_all || categories.contains(&"docker".to_string()) || categories.contains(&"setup".to_string()) { self.run_docker_setup_tests(&results).await?; } // Test 2: Vault connectivity if should_run_all || categories.contains(&"connectivity".to_string()) || categories.contains(&"basic".to_string()) { self.run_connectivity_tests(&results).await?; } // Test 3: Certificate lifecycle if should_run_all || categories.contains(&"certificates".to_string()) || categories.contains(&"lifecycle".to_string()) { self.run_certificate_tests(&results).await?; } // Test 4: Service integration if should_run_all || categories.contains(&"services".to_string()) || categories.contains(&"integration".to_string()) { self.run_service_integration_tests(&results).await?; } // Test 5: Failure scenarios if should_run_all || categories.contains(&"failures".to_string()) || categories.contains(&"resilience".to_string()) { self.run_failure_scenario_tests(&results).await?; } // Test 6: Performance validation if should_run_all || categories.contains(&"performance".to_string()) || categories.contains(&"hft".to_string()) { self.run_performance_tests(&results).await?; } // Test 7: Cleanup if self.config.cleanup_after_tests && (should_run_all || categories.contains(&"cleanup".to_string())) { self.run_cleanup_tests(&results).await?; } let final_results = results.read().await; info!("Test execution completed: {}/{} tests passed", final_results.success_count(), final_results.total_count()); Ok(final_results.clone()) } // Individual test category methods (similar to the main implementation) async fn run_docker_setup_tests(&self, results: &Arc>) -> Result<()> { info!("Running Docker environment setup tests"); let test_start = Instant::now(); match DockerEnvironment::new(&self.config).await { Ok(mut env) => { if let Err(e) = env.start_all_services().await { let mut test_results = results.write().await; test_results.add_failure("docker_environment_startup", e.to_string()); return Err(e); } let mut test_results = results.write().await; test_results.add_success("docker_environment_startup", test_start.elapsed()); Ok(()) } Err(e) => { let mut test_results = results.write().await; test_results.add_failure("docker_environment_setup", e.to_string()); Err(e) } } } async fn run_connectivity_tests(&self, results: &Arc>) -> Result<()> { info!("Running Vault connectivity tests"); match vault_connectivity_tests::run_connectivity_tests(&self.config).await { Ok(duration) => { let mut test_results = results.write().await; test_results.add_success("vault_connectivity", duration); Ok(()) } Err(e) => { let mut test_results = results.write().await; test_results.add_failure("vault_connectivity", e.to_string()); Err(e) } } } async fn run_certificate_tests(&self, results: &Arc>) -> Result<()> { certificate_lifecycle_tests::run_certificate_tests(&self.config, results).await } async fn run_service_integration_tests(&self, results: &Arc>) -> Result<()> { service_integration_tests::run_service_tests(&self.config, results).await } async fn run_failure_scenario_tests(&self, results: &Arc>) -> Result<()> { failure_scenario_tests::run_failure_tests(&self.config, results).await } async fn run_performance_tests(&self, results: &Arc>) -> Result<()> { performance_impact_tests::run_performance_tests(&self.config, results).await } async fn run_cleanup_tests(&self, results: &Arc>) -> Result<()> { let test_start = Instant::now(); if let Err(e) = DockerEnvironment::cleanup(&self.config).await { warn!("Docker cleanup failed: {}", e); } let mut test_results = results.write().await; test_results.add_success("cleanup", test_start.elapsed()); Ok(()) } } /// Generate comprehensive test reports async fn generate_test_reports(results: &VaultTestResults, output_dir: &str) -> Result<()> { info!("Generating test reports in: {}", output_dir); // Generate JSON report let json_report = serde_json::to_string_pretty(results) .context("Failed to serialize test results to JSON")?; let json_path = format!("{}/vault_e2e_results.json", output_dir); tokio::fs::write(&json_path, json_report).await .with_context(|| format!("Failed to write JSON report: {}", json_path))?; // Generate text report let text_report = results.generate_report(); let text_path = format!("{}/vault_e2e_results.txt", output_dir); tokio::fs::write(&text_path, text_report).await .with_context(|| format!("Failed to write text report: {}", text_path))?; // Generate HTML report (simplified) let html_report = generate_html_report(results); let html_path = format!("{}/vault_e2e_results.html", output_dir); tokio::fs::write(&html_path, html_report).await .with_context(|| format!("Failed to write HTML report: {}", html_path))?; info!("Test reports generated:"); info!(" JSON: {}", json_path); info!(" Text: {}", text_path); info!(" HTML: {}", html_path); Ok(()) } /// Generate HTML test report fn generate_html_report(results: &VaultTestResults) -> String { let success_rate = results.success_rate() * 100.0; let status_color = if success_rate >= 95.0 { "green" } else if success_rate >= 80.0 { "orange" } else { "red" }; format!(r#" Vault E2E Test Results

Vault E2E Integration Test Results

Success Rate: {success_rate:.1}%

Total Tests: {total}

Passed: {passed}

Failed: {failed}

Performance Metrics

Certificate Generation: {cert_time}
Cache Lookup: {cache_time}
TLS Handshake: {tls_time}
Memory Usage: {memory}
Cache Hit Rate: {hit_rate}

Test Results Details

{test_rows}
Test NameStatusDurationDetails
"#, status_color = status_color, success_rate = success_rate, total = results.total_count(), passed = results.success_count(), failed = results.failure_count(), cert_time = results.performance.cert_generation_time.map(|d| format!("{:?}", d)).unwrap_or("N/A".to_string()), cache_time = results.performance.cache_lookup_time.map(|d| format!("{:?}", d)).unwrap_or("N/A".to_string()), tls_time = results.performance.tls_handshake_time.map(|d| format!("{:?}", d)).unwrap_or("N/A".to_string()), memory = results.performance.memory_usage_mb.map(|m| format!("{:.1} MB", m)).unwrap_or("N/A".to_string()), hit_rate = results.performance.cache_hit_rate.map(|r| format!("{:.1}%", r)).unwrap_or("N/A".to_string()), test_rows = generate_test_table_rows(results), ) } fn generate_test_table_rows(results: &VaultTestResults) -> String { let mut rows = String::new(); for (name, duration) in &results.successes { rows.push_str(&format!( "{}PASSED{:?}-", name, duration )); } for (name, error) in &results.failures { rows.push_str(&format!( "{}FAILED-{}", name, error )); } rows } // Re-export the types and functions from the main module // Types are now defined directly in this file