foxhunt/testing/vault-integration/docker_compose.rs

//! Docker Compose environment management for Vault E2E tests
//!
//! This module provides comprehensive Docker environment management:
//! - Vault server with PKI secrets engine setup
//! - PostgreSQL and Redis for service dependencies
//! - ToxiProxy for network failure simulation
//! - Service health checking and startup coordination
//! - Environment cleanup and resource management

use anyhow::{Context, Result};
use std::collections::HashMap;
use std::path::Path;
use std::process::Command;
use std::time::{Duration, Instant};
use tokio::process::Command as AsyncCommand;
use tokio::time::{sleep, timeout};
use tracing::{debug, info, warn, error};

use crate::VaultTestConfig;

/// Docker Compose environment manager
pub struct DockerEnvironment {
    config: VaultTestConfig,
    compose_file: String,
    project_name: String,
    services: Vec<String>,
}

impl DockerEnvironment {
    /// Create new Docker environment
    pub async fn new(config: &VaultTestConfig) -> Result<Self> {
        let compose_file = "tests/e2e/vault_integration/docker-compose.vault.yml";
        
        // Verify compose file exists
        if !Path::new(compose_file).exists() {
            return Err(anyhow::anyhow!("Docker Compose file not found: {}", compose_file));
        }

        let services = vec![
            "vault".to_string(),
            "vault-init".to_string(),
            "postgres".to_string(),
            "redis".to_string(),
            "toxiproxy".to_string(),
        ];

        Ok(Self {
            config: config.clone(),
            compose_file: compose_file.to_string(),
            project_name: config.compose_project.clone(),
            services,
        })
    }

    /// Start all services with health checking
    pub async fn start_all_services(&mut self) -> Result<()> {
        info!("Starting Docker Compose services for Vault E2E testing");
        
        // Clean up any existing containers
        self.cleanup_existing().await?;

        // Start core infrastructure services first
        self.start_infrastructure_services().await?;

        // Wait for Vault initialization to complete
        self.wait_for_vault_setup().await?;

        // Start application services
        self.start_application_services().await?;

        info!("All Docker services started successfully");
        Ok(())
    }

    /// Start infrastructure services (Vault, PostgreSQL, Redis)
    async fn start_infrastructure_services(&self) -> Result<()> {
        info!("Starting infrastructure services");

        let infrastructure_services = ["vault", "postgres", "redis", "toxiproxy"];
        
        for service in &infrastructure_services {
            info!("Starting service: {}", service);
            
            let mut cmd = AsyncCommand::new("docker-compose");
            cmd.args([
                "-f", &self.compose_file,
                "-p", &self.project_name,
                "up", "-d", service
            ]);

            let output = cmd.output().await
                .with_context(|| format!("Failed to start service: {}", service))?;

            if !output.status.success() {
                let stderr = String::from_utf8_lossy(&output.stderr);
                return Err(anyhow::anyhow!(
                    "Failed to start service {}: {}", service, stderr
                ));
            }
        }

        // Wait for services to be healthy
        self.wait_for_service_health("vault", Duration::from_secs(30)).await?;
        self.wait_for_service_health("postgres", Duration::from_secs(20)).await?;
        self.wait_for_service_health("redis", Duration::from_secs(10)).await?;

        Ok(())
    }

    /// Wait for Vault initialization to complete
    async fn wait_for_vault_setup(&self) -> Result<()> {
        info!("Starting Vault initialization");

        // Start vault-init service
        let mut cmd = AsyncCommand::new("docker-compose");
        cmd.args([
            "-f", &self.compose_file,
            "-p", &self.project_name,
            "up", "vault-init"
        ]);

        let output = cmd.output().await
            .context("Failed to start vault-init service")?;

        if !output.status.success() {
            let stderr = String::from_utf8_lossy(&output.stderr);
            return Err(anyhow::anyhow!(
                "Vault initialization failed: {}", stderr
            ));
        }

        // Wait for initialization to complete
        self.wait_for_container_completion("vault-init", Duration::from_secs(60)).await?;

        // Verify Vault is properly configured
        self.verify_vault_configuration().await?;

        info!("Vault initialization completed successfully");
        Ok(())
    }

    /// Start application services (TLI, Trading Service)
    async fn start_application_services(&self) -> Result<()> {
        info!("Starting application services");

        let app_services = ["tli-service", "trading-service"];
        
        for service in &app_services {
            info!("Starting service: {}", service);
            
            let mut cmd = AsyncCommand::new("docker-compose");
            cmd.args([
                "-f", &self.compose_file,
                "-p", &self.project_name,
                "up", "-d", service
            ]);

            let output = cmd.output().await
                .with_context(|| format!("Failed to start service: {}", service))?;

            if !output.status.success() {
                let stderr = String::from_utf8_lossy(&output.stderr);
                warn!("Service {} failed to start: {}", service, stderr);
                // Continue with other services - app services may fail initially
            }
        }

        // Give application services time to start
        sleep(Duration::from_secs(10)).await;

        Ok(())
    }

    /// Wait for service to become healthy
    async fn wait_for_service_health(&self, service: &str, timeout_duration: Duration) -> Result<()> {
        info!("Waiting for service {} to become healthy", service);
        
        let start_time = Instant::now();
        let container_name = format!("foxhunt-{}-test", service);

        while start_time.elapsed() < timeout_duration {
            // Check container health status
            let mut cmd = AsyncCommand::new("docker");
            cmd.args(["inspect", "--format", "{{.State.Health.Status}}", &container_name]);

            match cmd.output().await {
                Ok(output) => {
                    let status = String::from_utf8_lossy(&output.stdout).trim().to_lowercase();
                    
                    if status == "healthy" {
                        info!("Service {} is healthy", service);
                        return Ok(());
                    } else if status == "unhealthy" {
                        return Err(anyhow::anyhow!("Service {} became unhealthy", service));
                    }
                }
                Err(e) => {
                    debug!("Health check failed for {}: {}", service, e);
                }
            }

            sleep(Duration::from_secs(2)).await;
        }

        Err(anyhow::anyhow!("Service {} did not become healthy within timeout", service))
    }

    /// Wait for container to complete execution
    async fn wait_for_container_completion(&self, service: &str, timeout_duration: Duration) -> Result<()> {
        info!("Waiting for container {} to complete", service);
        
        let start_time = Instant::now();
        let container_name = format!("foxhunt-{}", service);

        while start_time.elapsed() < timeout_duration {
            let mut cmd = AsyncCommand::new("docker");
            cmd.args(["inspect", "--format", "{{.State.Status}}", &container_name]);

            match cmd.output().await {
                Ok(output) => {
                    let status = String::from_utf8_lossy(&output.stdout).trim().to_lowercase();
                    
                    if status == "exited" {
                        // Check exit code
                        let mut exit_cmd = AsyncCommand::new("docker");
                        exit_cmd.args(["inspect", "--format", "{{.State.ExitCode}}", &container_name]);
                        
                        let exit_output = exit_cmd.output().await?;
                        let exit_code_str = String::from_utf8_lossy(&exit_output.stdout);
                        let exit_code = exit_code_str.trim();
                        
                        if exit_code == "0" {
                            info!("Container {} completed successfully", service);
                            return Ok(());
                        } else {
                            return Err(anyhow::anyhow!(
                                "Container {} exited with code {}", service, exit_code
                            ));
                        }
                    }
                }
                Err(e) => {
                    debug!("Status check failed for {}: {}", service, e);
                }
            }

            sleep(Duration::from_secs(2)).await;
        }

        Err(anyhow::anyhow!("Container {} did not complete within timeout", service))
    }

    /// Verify Vault configuration is correct
    async fn verify_vault_configuration(&self) -> Result<()> {
        info!("Verifying Vault configuration");

        // Check if PKI secrets engine is enabled
        let mut cmd = AsyncCommand::new("docker");
        cmd.args([
            "exec", "foxhunt-vault-test",
            "vault", "secrets", "list", "-format=json"
        ]);
        cmd.env("VAULT_ADDR", "http://localhost:8200");
        cmd.env("VAULT_TOKEN", "vault-root-token");

        let output = cmd.output().await
            .context("Failed to list Vault secrets engines")?;

        if !output.status.success() {
            let stderr = String::from_utf8_lossy(&output.stderr);
            return Err(anyhow::anyhow!("Failed to verify Vault secrets: {}", stderr));
        }

        let secrets_json = String::from_utf8_lossy(&output.stdout);
        if !secrets_json.contains("pki/") || !secrets_json.contains("pki_int/") {
            return Err(anyhow::anyhow!("PKI secrets engines not found"));
        }

        // Test certificate generation
        let mut cert_cmd = AsyncCommand::new("docker");
        cert_cmd.args([
            "exec", "foxhunt-vault-test",
            "vault", "write", "-format=json",
            "pki_int/issue/hft-trading",
            "common_name=test.foxhunt.internal",
            "ttl=1h"
        ]);
        cert_cmd.env("VAULT_ADDR", "http://localhost:8200");
        cert_cmd.env("VAULT_TOKEN", "vault-root-token");

        let cert_output = cert_cmd.output().await
            .context("Failed to test certificate generation")?;

        if !cert_output.status.success() {
            let stderr = String::from_utf8_lossy(&cert_output.stderr);
            return Err(anyhow::anyhow!("Certificate generation test failed: {}", stderr));
        }

        info!("Vault configuration verified successfully");
        Ok(())
    }

    /// Get service logs for debugging
    pub async fn get_service_logs(&self, service: &str) -> Result<String> {
        let mut cmd = AsyncCommand::new("docker-compose");
        cmd.args([
            "-f", &self.compose_file,
            "-p", &self.project_name,
            "logs", service
        ]);

        let output = cmd.output().await
            .with_context(|| format!("Failed to get logs for service: {}", service))?;

        Ok(String::from_utf8_lossy(&output.stdout).to_string())
    }

    /// Stop specific service
    pub async fn stop_service(&self, service: &str) -> Result<()> {
        info!("Stopping service: {}", service);

        let mut cmd = AsyncCommand::new("docker-compose");
        cmd.args([
            "-f", &self.compose_file,
            "-p", &self.project_name,
            "stop", service
        ]);

        let output = cmd.output().await
            .with_context(|| format!("Failed to stop service: {}", service))?;

        if !output.status.success() {
            let stderr = String::from_utf8_lossy(&output.stderr);
            return Err(anyhow::anyhow!(
                "Failed to stop service {}: {}", service, stderr
            ));
        }

        Ok(())
    }

    /// Start specific service
    pub async fn start_service(&self, service: &str) -> Result<()> {
        info!("Starting service: {}", service);

        let mut cmd = AsyncCommand::new("docker-compose");
        cmd.args([
            "-f", &self.compose_file,
            "-p", &self.project_name,
            "start", service
        ]);

        let output = cmd.output().await
            .with_context(|| format!("Failed to start service: {}", service))?;

        if !output.status.success() {
            let stderr = String::from_utf8_lossy(&output.stderr);
            return Err(anyhow::anyhow!(
                "Failed to start service {}: {}", service, stderr
            ));
        }

        Ok(())
    }

    /// Simulate network partition using ToxiProxy
    pub async fn simulate_network_partition(&self, target_service: &str) -> Result<()> {
        info!("Simulating network partition for service: {}", target_service);

        // Add latency and packet loss toxic
        let mut cmd = AsyncCommand::new("curl");
        cmd.args([
            "-X", "POST",
            "http://localhost:8474/proxies/vault-proxy/toxics",
            "-H", "Content-Type: application/json",
            "-d", r#"{"name":"latency","type":"latency","attributes":{"latency":5000}}"#
        ]);

        let output = cmd.output().await
            .context("Failed to add network latency toxic")?;

        if !output.status.success() {
            let stderr = String::from_utf8_lossy(&output.stderr);
            return Err(anyhow::anyhow!("Failed to add network toxic: {}", stderr));
        }

        Ok(())
    }

    /// Remove network partition simulation
    pub async fn remove_network_partition(&self) -> Result<()> {
        info!("Removing network partition simulation");

        let mut cmd = AsyncCommand::new("curl");
        cmd.args([
            "-X", "DELETE",
            "http://localhost:8474/proxies/vault-proxy/toxics/latency"
        ]);

        let output = cmd.output().await
            .context("Failed to remove network toxic")?;

        if !output.status.success() {
            let stderr = String::from_utf8_lossy(&output.stderr);
            warn!("Failed to remove network toxic: {}", stderr);
        }

        Ok(())
    }

    /// Clean up existing containers
    async fn cleanup_existing(&self) -> Result<()> {
        info!("Cleaning up existing containers");

        let mut cmd = AsyncCommand::new("docker-compose");
        cmd.args([
            "-f", &self.compose_file,
            "-p", &self.project_name,
            "down", "-v", "--remove-orphans"
        ]);

        let output = cmd.output().await
            .context("Failed to cleanup existing containers")?;

        if !output.status.success() {
            let stderr = String::from_utf8_lossy(&output.stderr);
            warn!("Cleanup warning: {}", stderr);
        }

        Ok(())
    }

    /// Cleanup all resources
    pub async fn cleanup(&config: &VaultTestConfig) -> Result<()> {
        info!("Cleaning up Docker Compose resources");

        let compose_file = "tests/e2e/vault_integration/docker-compose.vault.yml";
        
        let mut cmd = AsyncCommand::new("docker-compose");
        cmd.args([
            "-f", compose_file,
            "-p", &config.compose_project,
            "down", "-v", "--remove-orphans", "--rmi", "local"
        ]);

        let output = cmd.output().await
            .context("Failed to cleanup Docker resources")?;

        if !output.status.success() {
            let stderr = String::from_utf8_lossy(&output.stderr);
            warn!("Cleanup completed with warnings: {}", stderr);
        }

        // Remove any lingering test certificates
        if Path::new(&config.cert_cache_dir).exists() {
            std::fs::remove_dir_all(&config.cert_cache_dir)
                .with_context(|| format!("Failed to remove cert cache dir: {}", config.cert_cache_dir))?;
        }

        info!("Docker cleanup completed");
        Ok(())
    }

    /// Get container statistics
    pub async fn get_container_stats(&self) -> Result<HashMap<String, serde_json::Value>> {
        let mut stats = HashMap::new();

        for service in &self.services {
            let container_name = format!("foxhunt-{}-test", service);
            
            let mut cmd = AsyncCommand::new("docker");
            cmd.args([
                "stats", "--no-stream", "--format",
                "{{json .}}", &container_name
            ]);

            match cmd.output().await {
                Ok(output) => {
                    if output.status.success() {
                        let stats_json = String::from_utf8_lossy(&output.stdout);
                        if let Ok(parsed) = serde_json::from_str::<serde_json::Value>(&stats_json) {
                            stats.insert(service.clone(), parsed);
                        }
                    }
                }
                Err(e) => {
                    debug!("Failed to get stats for {}: {}", service, e);
                }
            }
        }

        Ok(stats)
    }
}

impl Drop for DockerEnvironment {
    fn drop(&mut self) {
        if self.config.cleanup_after_tests {
            // Spawn cleanup task (best effort)
            tokio::spawn(async move {
                if let Err(e) = DockerEnvironment::cleanup(&self.config).await {
                    warn!("Background cleanup failed: {}", e);
                }
            });
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    #[ignore = "Requires Docker"]
    async fn test_docker_environment_creation() {
        let config = VaultTestConfig::default();
        let env = DockerEnvironment::new(&config).await.unwrap();
        assert_eq!(env.project_name, config.compose_project);
        assert!(env.services.contains(&"vault".to_string()));
    }

    #[test]
    fn test_cleanup_config() {
        let mut config = VaultTestConfig::default();
        config.cleanup_after_tests = false;
        
        // Should not cleanup when disabled
        assert!(!config.cleanup_after_tests);
    }
}