//! Docker Compose environment management for Vault E2E tests //! //! This module provides comprehensive Docker environment management: //! - Vault server with PKI secrets engine setup //! - PostgreSQL and Redis for service dependencies //! - ToxiProxy for network failure simulation //! - Service health checking and startup coordination //! - Environment cleanup and resource management use anyhow::{Context, Result}; use std::collections::HashMap; use std::path::Path; use std::process::Command; use std::time::{Duration, Instant}; use tokio::process::Command as AsyncCommand; use tokio::time::{sleep, timeout}; use tracing::{debug, info, warn, error}; use crate::VaultTestConfig; /// Docker Compose environment manager pub struct DockerEnvironment { config: VaultTestConfig, compose_file: String, project_name: String, services: Vec, } impl DockerEnvironment { /// Create new Docker environment pub async fn new(config: &VaultTestConfig) -> Result { let compose_file = "tests/e2e/vault_integration/docker-compose.vault.yml"; // Verify compose file exists if !Path::new(compose_file).exists() { return Err(anyhow::anyhow!("Docker Compose file not found: {}", compose_file)); } let services = vec![ "vault".to_string(), "vault-init".to_string(), "postgres".to_string(), "redis".to_string(), "toxiproxy".to_string(), ]; Ok(Self { config: config.clone(), compose_file: compose_file.to_string(), project_name: config.compose_project.clone(), services, }) } /// Start all services with health checking pub async fn start_all_services(&mut self) -> Result<()> { info!("Starting Docker Compose services for Vault E2E testing"); // Clean up any existing containers self.cleanup_existing().await?; // Start core infrastructure services first self.start_infrastructure_services().await?; // Wait for Vault initialization to complete self.wait_for_vault_setup().await?; // Start application services self.start_application_services().await?; info!("All Docker services started successfully"); Ok(()) } /// Start infrastructure services (Vault, PostgreSQL, Redis) async fn start_infrastructure_services(&self) -> Result<()> { info!("Starting infrastructure services"); let infrastructure_services = ["vault", "postgres", "redis", "toxiproxy"]; for service in &infrastructure_services { info!("Starting service: {}", service); let mut cmd = AsyncCommand::new("docker-compose"); cmd.args([ "-f", &self.compose_file, "-p", &self.project_name, "up", "-d", service ]); let output = cmd.output().await .with_context(|| format!("Failed to start service: {}", service))?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); return Err(anyhow::anyhow!( "Failed to start service {}: {}", service, stderr )); } } // Wait for services to be healthy self.wait_for_service_health("vault", Duration::from_secs(30)).await?; self.wait_for_service_health("postgres", Duration::from_secs(20)).await?; self.wait_for_service_health("redis", Duration::from_secs(10)).await?; Ok(()) } /// Wait for Vault initialization to complete async fn wait_for_vault_setup(&self) -> Result<()> { info!("Starting Vault initialization"); // Start vault-init service let mut cmd = AsyncCommand::new("docker-compose"); cmd.args([ "-f", &self.compose_file, "-p", &self.project_name, "up", "vault-init" ]); let output = cmd.output().await .context("Failed to start vault-init service")?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); return Err(anyhow::anyhow!( "Vault initialization failed: {}", stderr )); } // Wait for initialization to complete self.wait_for_container_completion("vault-init", Duration::from_secs(60)).await?; // Verify Vault is properly configured self.verify_vault_configuration().await?; info!("Vault initialization completed successfully"); Ok(()) } /// Start application services (TLI, Trading Service) async fn start_application_services(&self) -> Result<()> { info!("Starting application services"); let app_services = ["tli-service", "trading-service"]; for service in &app_services { info!("Starting service: {}", service); let mut cmd = AsyncCommand::new("docker-compose"); cmd.args([ "-f", &self.compose_file, "-p", &self.project_name, "up", "-d", service ]); let output = cmd.output().await .with_context(|| format!("Failed to start service: {}", service))?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); warn!("Service {} failed to start: {}", service, stderr); // Continue with other services - app services may fail initially } } // Give application services time to start sleep(Duration::from_secs(10)).await; Ok(()) } /// Wait for service to become healthy async fn wait_for_service_health(&self, service: &str, timeout_duration: Duration) -> Result<()> { info!("Waiting for service {} to become healthy", service); let start_time = Instant::now(); let container_name = format!("foxhunt-{}-test", service); while start_time.elapsed() < timeout_duration { // Check container health status let mut cmd = AsyncCommand::new("docker"); cmd.args(["inspect", "--format", "{{.State.Health.Status}}", &container_name]); match cmd.output().await { Ok(output) => { let status = String::from_utf8_lossy(&output.stdout).trim().to_lowercase(); if status == "healthy" { info!("Service {} is healthy", service); return Ok(()); } else if status == "unhealthy" { return Err(anyhow::anyhow!("Service {} became unhealthy", service)); } } Err(e) => { debug!("Health check failed for {}: {}", service, e); } } sleep(Duration::from_secs(2)).await; } Err(anyhow::anyhow!("Service {} did not become healthy within timeout", service)) } /// Wait for container to complete execution async fn wait_for_container_completion(&self, service: &str, timeout_duration: Duration) -> Result<()> { info!("Waiting for container {} to complete", service); let start_time = Instant::now(); let container_name = format!("foxhunt-{}", service); while start_time.elapsed() < timeout_duration { let mut cmd = AsyncCommand::new("docker"); cmd.args(["inspect", "--format", "{{.State.Status}}", &container_name]); match cmd.output().await { Ok(output) => { let status = String::from_utf8_lossy(&output.stdout).trim().to_lowercase(); if status == "exited" { // Check exit code let mut exit_cmd = AsyncCommand::new("docker"); exit_cmd.args(["inspect", "--format", "{{.State.ExitCode}}", &container_name]); let exit_output = exit_cmd.output().await?; let exit_code_str = String::from_utf8_lossy(&exit_output.stdout); let exit_code = exit_code_str.trim(); if exit_code == "0" { info!("Container {} completed successfully", service); return Ok(()); } else { return Err(anyhow::anyhow!( "Container {} exited with code {}", service, exit_code )); } } } Err(e) => { debug!("Status check failed for {}: {}", service, e); } } sleep(Duration::from_secs(2)).await; } Err(anyhow::anyhow!("Container {} did not complete within timeout", service)) } /// Verify Vault configuration is correct async fn verify_vault_configuration(&self) -> Result<()> { info!("Verifying Vault configuration"); // Check if PKI secrets engine is enabled let mut cmd = AsyncCommand::new("docker"); cmd.args([ "exec", "foxhunt-vault-test", "vault", "secrets", "list", "-format=json" ]); cmd.env("VAULT_ADDR", "http://localhost:8200"); cmd.env("VAULT_TOKEN", "vault-root-token"); let output = cmd.output().await .context("Failed to list Vault secrets engines")?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); return Err(anyhow::anyhow!("Failed to verify Vault secrets: {}", stderr)); } let secrets_json = String::from_utf8_lossy(&output.stdout); if !secrets_json.contains("pki/") || !secrets_json.contains("pki_int/") { return Err(anyhow::anyhow!("PKI secrets engines not found")); } // Test certificate generation let mut cert_cmd = AsyncCommand::new("docker"); cert_cmd.args([ "exec", "foxhunt-vault-test", "vault", "write", "-format=json", "pki_int/issue/hft-trading", "common_name=test.foxhunt.internal", "ttl=1h" ]); cert_cmd.env("VAULT_ADDR", "http://localhost:8200"); cert_cmd.env("VAULT_TOKEN", "vault-root-token"); let cert_output = cert_cmd.output().await .context("Failed to test certificate generation")?; if !cert_output.status.success() { let stderr = String::from_utf8_lossy(&cert_output.stderr); return Err(anyhow::anyhow!("Certificate generation test failed: {}", stderr)); } info!("Vault configuration verified successfully"); Ok(()) } /// Get service logs for debugging pub async fn get_service_logs(&self, service: &str) -> Result { let mut cmd = AsyncCommand::new("docker-compose"); cmd.args([ "-f", &self.compose_file, "-p", &self.project_name, "logs", service ]); let output = cmd.output().await .with_context(|| format!("Failed to get logs for service: {}", service))?; Ok(String::from_utf8_lossy(&output.stdout).to_string()) } /// Stop specific service pub async fn stop_service(&self, service: &str) -> Result<()> { info!("Stopping service: {}", service); let mut cmd = AsyncCommand::new("docker-compose"); cmd.args([ "-f", &self.compose_file, "-p", &self.project_name, "stop", service ]); let output = cmd.output().await .with_context(|| format!("Failed to stop service: {}", service))?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); return Err(anyhow::anyhow!( "Failed to stop service {}: {}", service, stderr )); } Ok(()) } /// Start specific service pub async fn start_service(&self, service: &str) -> Result<()> { info!("Starting service: {}", service); let mut cmd = AsyncCommand::new("docker-compose"); cmd.args([ "-f", &self.compose_file, "-p", &self.project_name, "start", service ]); let output = cmd.output().await .with_context(|| format!("Failed to start service: {}", service))?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); return Err(anyhow::anyhow!( "Failed to start service {}: {}", service, stderr )); } Ok(()) } /// Simulate network partition using ToxiProxy pub async fn simulate_network_partition(&self, target_service: &str) -> Result<()> { info!("Simulating network partition for service: {}", target_service); // Add latency and packet loss toxic let mut cmd = AsyncCommand::new("curl"); cmd.args([ "-X", "POST", "http://localhost:8474/proxies/vault-proxy/toxics", "-H", "Content-Type: application/json", "-d", r#"{"name":"latency","type":"latency","attributes":{"latency":5000}}"# ]); let output = cmd.output().await .context("Failed to add network latency toxic")?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); return Err(anyhow::anyhow!("Failed to add network toxic: {}", stderr)); } Ok(()) } /// Remove network partition simulation pub async fn remove_network_partition(&self) -> Result<()> { info!("Removing network partition simulation"); let mut cmd = AsyncCommand::new("curl"); cmd.args([ "-X", "DELETE", "http://localhost:8474/proxies/vault-proxy/toxics/latency" ]); let output = cmd.output().await .context("Failed to remove network toxic")?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); warn!("Failed to remove network toxic: {}", stderr); } Ok(()) } /// Clean up existing containers async fn cleanup_existing(&self) -> Result<()> { info!("Cleaning up existing containers"); let mut cmd = AsyncCommand::new("docker-compose"); cmd.args([ "-f", &self.compose_file, "-p", &self.project_name, "down", "-v", "--remove-orphans" ]); let output = cmd.output().await .context("Failed to cleanup existing containers")?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); warn!("Cleanup warning: {}", stderr); } Ok(()) } /// Cleanup all resources pub async fn cleanup(&config: &VaultTestConfig) -> Result<()> { info!("Cleaning up Docker Compose resources"); let compose_file = "tests/e2e/vault_integration/docker-compose.vault.yml"; let mut cmd = AsyncCommand::new("docker-compose"); cmd.args([ "-f", compose_file, "-p", &config.compose_project, "down", "-v", "--remove-orphans", "--rmi", "local" ]); let output = cmd.output().await .context("Failed to cleanup Docker resources")?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); warn!("Cleanup completed with warnings: {}", stderr); } // Remove any lingering test certificates if Path::new(&config.cert_cache_dir).exists() { std::fs::remove_dir_all(&config.cert_cache_dir) .with_context(|| format!("Failed to remove cert cache dir: {}", config.cert_cache_dir))?; } info!("Docker cleanup completed"); Ok(()) } /// Get container statistics pub async fn get_container_stats(&self) -> Result> { let mut stats = HashMap::new(); for service in &self.services { let container_name = format!("foxhunt-{}-test", service); let mut cmd = AsyncCommand::new("docker"); cmd.args([ "stats", "--no-stream", "--format", "{{json .}}", &container_name ]); match cmd.output().await { Ok(output) => { if output.status.success() { let stats_json = String::from_utf8_lossy(&output.stdout); if let Ok(parsed) = serde_json::from_str::(&stats_json) { stats.insert(service.clone(), parsed); } } } Err(e) => { debug!("Failed to get stats for {}: {}", service, e); } } } Ok(stats) } } impl Drop for DockerEnvironment { fn drop(&mut self) { if self.config.cleanup_after_tests { // Spawn cleanup task (best effort) tokio::spawn(async move { if let Err(e) = DockerEnvironment::cleanup(&self.config).await { warn!("Background cleanup failed: {}", e); } }); } } } #[cfg(test)] mod tests { use super::*; #[tokio::test] #[ignore = "Requires Docker"] async fn test_docker_environment_creation() { let config = VaultTestConfig::default(); let env = DockerEnvironment::new(&config).await.unwrap(); assert_eq!(env.project_name, config.compose_project); assert!(env.services.contains(&"vault".to_string())); } #[test] fn test_cleanup_config() { let mut config = VaultTestConfig::default(); config.cleanup_after_tests = false; // Should not cleanup when disabled assert!(!config.cleanup_after_tests); } }