foxhunt/testing/vault-integration/service_integration_tests.rs

//! Service integration tests with Vault dependencies
//!
//! This module tests how services integrate with Vault:
//! - Service startup with Vault dependencies
//! - Certificate provisioning during service initialization
//! - Inter-service mTLS communication with Vault certificates
//! - Configuration hot-reload with Vault-backed secrets
//! - Service health monitoring with Vault integration
//! - Graceful degradation when Vault is unavailable

use anyhow::{Context, Result};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::process::Command as AsyncCommand;
use tokio::sync::RwLock;
use tokio::time::{sleep, timeout};
use tracing::{debug, info, warn, error};

use crate::{VaultTestConfig, VaultTestResults};

/// Service status information
#[derive(Debug, Clone, PartialEq)]
pub enum ServiceStatus {
    NotStarted,
    Starting,
    Healthy,
    Unhealthy,
    Stopped,
}

/// Service information for testing
#[derive(Debug, Clone)]
pub struct ServiceInfo {
    pub name: String,
    pub container_name: String,
    pub health_check_url: Option<String>,
    pub expected_startup_time: Duration,
    pub requires_vault: bool,
    pub status: ServiceStatus,
}

impl ServiceInfo {
    pub fn new(name: &str, requires_vault: bool) -> Self {
        Self {
            name: name.to_string(),
            container_name: format!("foxhunt-{}-test", name),
            health_check_url: Self::get_health_url(name),
            expected_startup_time: Duration::from_secs(30),
            requires_vault,
            status: ServiceStatus::NotStarted,
        }
    }

    fn get_health_url(service: &str) -> Option<String> {
        match service {
            "tli-service" => Some("http://localhost:3000/health".to_string()),
            "trading-service" => Some("http://localhost:50052/health".to_string()),
            _ => None,
        }
    }
}

/// Service integration test coordinator
pub struct ServiceIntegrationTester {
    config: VaultTestConfig,
    services: Vec<ServiceInfo>,
}

impl ServiceIntegrationTester {
    pub fn new(config: VaultTestConfig) -> Self {
        let services = vec![
            ServiceInfo::new("tli-service", true),
            ServiceInfo::new("trading-service", true),
        ];

        Self { config, services }
    }

    /// Test service startup sequence with Vault dependencies
    pub async fn test_service_startup_sequence(
        &mut self,
        results: &Arc<RwLock<VaultTestResults>>,
    ) -> Result<()> {
        info!("Testing service startup sequence with Vault dependencies");

        let overall_start = Instant::now();

        // Test 1: Start services in dependency order
        for service_info in &mut self.services {
            if service_info.requires_vault {
                info!("Starting Vault-dependent service: {}", service_info.name);
                
                let startup_start = Instant::now();
                self.start_service(&service_info.name).await?;
                
                // Wait for service to become healthy
                let health_result = self.wait_for_service_health(
                    service_info,
                    service_info.expected_startup_time,
                ).await;

                match health_result {
                    Ok(_) => {
                        let startup_duration = startup_start.elapsed();
                        service_info.status = ServiceStatus::Healthy;
                        
                        let mut test_results = results.write().await;
                        test_results.add_success(
                            &format!("{}_startup", service_info.name),
                            startup_duration,
                        );
                        
                        info!("Service {} started successfully in {:?}", 
                              service_info.name, startup_duration);
                    }
                    Err(e) => {
                        service_info.status = ServiceStatus::Unhealthy;
                        
                        let mut test_results = results.write().await;
                        test_results.add_failure(
                            &format!("{}_startup", service_info.name),
                            e.to_string(),
                        );
                        
                        warn!("Service {} failed to start: {}", service_info.name, e);
                    }
                }
            }
        }

        let total_startup_time = overall_start.elapsed();
        
        {
            let mut test_results = results.write().await;
            test_results.add_success("service_startup_sequence", total_startup_time);
        }

        info!("Service startup sequence completed in {:?}", total_startup_time);
        Ok(())
    }

    /// Test certificate provisioning during service startup
    pub async fn test_certificate_provisioning(
        &self,
        results: &Arc<RwLock<VaultTestResults>>,
    ) -> Result<()> {
        info!("Testing certificate provisioning during service startup");

        let test_start = Instant::now();

        // Check if services have provisioned certificates from Vault
        for service_info in &self.services {
            if service_info.status == ServiceStatus::Healthy && service_info.requires_vault {
                // Check service logs for certificate provisioning
                let logs = self.get_service_logs(&service_info.name).await?;
                
                if logs.contains("Certificate obtained from Vault") || 
                   logs.contains("Successfully connected to Vault") {
                    info!("Service {} successfully provisioned certificates", service_info.name);
                } else {
                    warn!("Service {} may not have provisioned certificates from Vault", 
                          service_info.name);
                }

                // Check if certificate files exist in the container
                self.verify_certificate_files(&service_info.container_name).await?;
            }
        }

        let provisioning_duration = test_start.elapsed();
        
        {
            let mut test_results = results.write().await;
            test_results.add_success("certificate_provisioning", provisioning_duration);
        }

        info!("Certificate provisioning test completed in {:?}", provisioning_duration);
        Ok(())
    }

    /// Test inter-service mTLS communication
    pub async fn test_inter_service_mtls(
        &self,
        results: &Arc<RwLock<VaultTestResults>>,
    ) -> Result<()> {
        info!("Testing inter-service mTLS communication");

        let test_start = Instant::now();

        // Find healthy services for communication testing
        let healthy_services: Vec<_> = self.services.iter()
            .filter(|s| s.status == ServiceStatus::Healthy)
            .collect();

        if healthy_services.len() < 2 {
            let mut test_results = results.write().await;
            test_results.add_failure(
                "inter_service_mtls",
                "Not enough healthy services for mTLS testing".to_string(),
            );
            return Ok(());
        }

        // Test TLI -> Trading Service communication
        let mtls_result = self.test_grpc_communication().await;

        match mtls_result {
            Ok(comm_duration) => {
                let mut test_results = results.write().await;
                test_results.add_success("inter_service_mtls", comm_duration);
                test_results.performance.tls_handshake_time = Some(comm_duration);
                
                info!("Inter-service mTLS communication successful");
            }
            Err(e) => {
                let mut test_results = results.write().await;
                test_results.add_failure("inter_service_mtls", e.to_string());
                
                warn!("Inter-service mTLS communication failed: {}", e);
            }
        }

        let total_duration = test_start.elapsed();
        info!("Inter-service mTLS test completed in {:?}", total_duration);
        Ok(())
    }

    /// Test configuration hot-reload with Vault secrets
    pub async fn test_configuration_hot_reload(
        &self,
        results: &Arc<RwLock<VaultTestResults>>,
    ) -> Result<()> {
        info!("Testing configuration hot-reload with Vault secrets");

        let test_start = Instant::now();

        // Update a configuration value in PostgreSQL (simulating config change)
        self.update_test_configuration().await?;

        // Wait for services to pick up the change
        sleep(Duration::from_secs(5)).await;

        // Check service logs for configuration reload
        let mut reload_detected = false;
        
        for service_info in &self.services {
            if service_info.status == ServiceStatus::Healthy {
                let logs = self.get_service_logs(&service_info.name).await?;
                
                if logs.contains("Configuration reloaded") || 
                   logs.contains("Hot reload triggered") {
                    reload_detected = true;
                    info!("Service {} detected configuration hot-reload", service_info.name);
                    break;
                }
            }
        }

        let reload_duration = test_start.elapsed();

        if reload_detected {
            let mut test_results = results.write().await;
            test_results.add_success("configuration_hot_reload", reload_duration);
        } else {
            let mut test_results = results.write().await;
            test_results.add_failure(
                "configuration_hot_reload",
                "No services detected configuration hot-reload".to_string(),
            );
        }

        info!("Configuration hot-reload test completed in {:?}", reload_duration);
        Ok(())
    }

    /// Test graceful degradation when Vault is unavailable
    pub async fn test_vault_unavailable_degradation(
        &self,
        results: &Arc<RwLock<VaultTestResults>>,
    ) -> Result<()> {
        info!("Testing graceful degradation when Vault is unavailable");

        let test_start = Instant::now();

        // Stop Vault service temporarily
        self.stop_vault_service().await?;

        // Wait for services to detect Vault unavailability
        sleep(Duration::from_secs(10)).await;

        // Check that services continue running and use cached certificates
        let mut services_degraded_gracefully = 0;

        for service_info in &self.services {
            if service_info.status == ServiceStatus::Healthy && service_info.requires_vault {
                let service_still_healthy = self.check_service_health(&service_info.name).await.is_ok();
                
                if service_still_healthy {
                    services_degraded_gracefully += 1;
                    info!("Service {} degraded gracefully without Vault", service_info.name);
                } else {
                    warn!("Service {} failed when Vault became unavailable", service_info.name);
                }

                // Check logs for circuit breaker activation
                let logs = self.get_service_logs(&service_info.name).await?;
                if logs.contains("Circuit breaker opened") || 
                   logs.contains("Using cached certificates") {
                    info!("Service {} activated circuit breaker for Vault", service_info.name);
                }
            }
        }

        // Restart Vault service
        self.start_vault_service().await?;

        let degradation_duration = test_start.elapsed();

        if services_degraded_gracefully > 0 {
            let mut test_results = results.write().await;
            test_results.add_success("vault_unavailable_degradation", degradation_duration);
        } else {
            let mut test_results = results.write().await;
            test_results.add_failure(
                "vault_unavailable_degradation",
                "No services degraded gracefully".to_string(),
            );
        }

        info!("Vault unavailability degradation test completed in {:?}", degradation_duration);
        Ok(())
    }

    /// Start a specific service
    async fn start_service(&self, service_name: &str) -> Result<()> {
        let mut cmd = AsyncCommand::new("docker-compose");
        cmd.args([
            "-f", "tests/e2e/vault_integration/docker-compose.vault.yml",
            "-p", &self.config.compose_project,
            "up", "-d", service_name
        ]);

        let output = cmd.output().await
            .with_context(|| format!("Failed to start service: {}", service_name))?;

        if !output.status.success() {
            let stderr = String::from_utf8_lossy(&output.stderr);
            return Err(anyhow::anyhow!(
                "Failed to start service {}: {}", service_name, stderr
            ));
        }

        Ok(())
    }

    /// Wait for service to become healthy
    async fn wait_for_service_health(
        &self,
        service_info: &ServiceInfo,
        timeout_duration: Duration,
    ) -> Result<()> {
        let start_time = Instant::now();

        while start_time.elapsed() < timeout_duration {
            match self.check_service_health(&service_info.name).await {
                Ok(_) => return Ok(()),
                Err(_) => {
                    sleep(Duration::from_secs(2)).await;
                }
            }
        }

        Err(anyhow::anyhow!(
            "Service {} did not become healthy within {:?}",
            service_info.name, timeout_duration
        ))
    }

    /// Check if a service is healthy
    async fn check_service_health(&self, service_name: &str) -> Result<()> {
        // Check Docker container health
        let container_name = format!("foxhunt-{}-test", service_name);
        
        let mut cmd = AsyncCommand::new("docker");
        cmd.args(["inspect", "--format", "{{.State.Health.Status}}", &container_name]);

        let output = cmd.output().await?;
        let status = String::from_utf8_lossy(&output.stdout).trim().to_lowercase();

        if status == "healthy" {
            Ok(())
        } else if status == "unhealthy" {
            Err(anyhow::anyhow!("Service {} is unhealthy", service_name))
        } else {
            Err(anyhow::anyhow!("Service {} health status unknown: {}", service_name, status))
        }
    }

    /// Get service logs
    async fn get_service_logs(&self, service_name: &str) -> Result<String> {
        let mut cmd = AsyncCommand::new("docker-compose");
        cmd.args([
            "-f", "tests/e2e/vault_integration/docker-compose.vault.yml",
            "-p", &self.config.compose_project,
            "logs", "--tail", "100", service_name
        ]);

        let output = cmd.output().await
            .with_context(|| format!("Failed to get logs for service: {}", service_name))?;

        Ok(String::from_utf8_lossy(&output.stdout).to_string())
    }

    /// Verify certificate files exist in container
    async fn verify_certificate_files(&self, container_name: &str) -> Result<()> {
        let cert_files = [
            "/opt/foxhunt/certs/service.crt",
            "/opt/foxhunt/certs/service.key",
            "/opt/foxhunt/certs/ca.crt",
        ];

        for cert_file in &cert_files {
            let mut cmd = AsyncCommand::new("docker");
            cmd.args(["exec", container_name, "test", "-f", cert_file]);

            let output = cmd.output().await?;
            
            if !output.status.success() {
                return Err(anyhow::anyhow!(
                    "Certificate file {} not found in container {}", cert_file, container_name
                ));
            }
        }

        info!("All certificate files verified in container: {}", container_name);
        Ok(())
    }

    /// Test gRPC communication between services
    async fn test_grpc_communication(&self) -> Result<Duration> {
        let start = Instant::now();
        
        // Simplified gRPC health check - in real implementation would use actual gRPC client
        let mut cmd = AsyncCommand::new("curl");
        cmd.args([
            "-s", "-f", "--max-time", "5",
            "http://localhost:50051/health"  // TLI service health endpoint
        ]);

        let output = cmd.output().await
            .context("Failed to perform gRPC health check")?;

        if output.status.success() {
            Ok(start.elapsed())
        } else {
            Err(anyhow::anyhow!("gRPC communication test failed"))
        }
    }

    /// Update test configuration in PostgreSQL
    async fn update_test_configuration(&self) -> Result<()> {
        let mut cmd = AsyncCommand::new("docker");
        cmd.args([
            "exec", "foxhunt-postgres-test",
            "psql", "-U", "foxhunt", "-d", "foxhunt_test",
            "-c", "UPDATE configuration SET value = 'test_hot_reload_' || extract(epoch from now()) WHERE key = 'test_config';"
        ]);

        let output = cmd.output().await
            .context("Failed to update test configuration")?;

        if !output.status.success() {
            let stderr = String::from_utf8_lossy(&output.stderr);
            return Err(anyhow::anyhow!("Failed to update configuration: {}", stderr));
        }

        // Trigger PostgreSQL NOTIFY for hot reload
        let mut notify_cmd = AsyncCommand::new("docker");
        notify_cmd.args([
            "exec", "foxhunt-postgres-test",
            "psql", "-U", "foxhunt", "-d", "foxhunt_test",
            "-c", "NOTIFY config_update;"
        ]);

        notify_cmd.output().await?;
        Ok(())
    }

    /// Stop Vault service for testing degradation
    async fn stop_vault_service(&self) -> Result<()> {
        let mut cmd = AsyncCommand::new("docker-compose");
        cmd.args([
            "-f", "tests/e2e/vault_integration/docker-compose.vault.yml",
            "-p", &self.config.compose_project,
            "stop", "vault"
        ]);

        cmd.output().await
            .context("Failed to stop Vault service")?;

        info!("Vault service stopped for degradation testing");
        Ok(())
    }

    /// Start Vault service after testing
    async fn start_vault_service(&self) -> Result<()> {
        let mut cmd = AsyncCommand::new("docker-compose");
        cmd.args([
            "-f", "tests/e2e/vault_integration/docker-compose.vault.yml",
            "-p", &self.config.compose_project,
            "start", "vault"
        ]);

        cmd.output().await
            .context("Failed to start Vault service")?;

        // Wait for Vault to become healthy again
        sleep(Duration::from_secs(10)).await;

        info!("Vault service restarted");
        Ok(())
    }
}

/// Run all service integration tests
pub async fn run_service_tests(
    config: &VaultTestConfig,
    results: &Arc<RwLock<VaultTestResults>>,
) -> Result<()> {
    info!("Running service integration tests");

    let mut tester = ServiceIntegrationTester::new(config.clone());

    // Test 1: Service startup sequence
    tester.test_service_startup_sequence(results).await?;

    // Test 2: Certificate provisioning
    tester.test_certificate_provisioning(results).await?;

    // Test 3: Inter-service mTLS communication
    tester.test_inter_service_mtls(results).await?;

    // Test 4: Configuration hot-reload
    tester.test_configuration_hot_reload(results).await?;

    // Test 5: Graceful degradation
    tester.test_vault_unavailable_degradation(results).await?;

    info!("All service integration tests completed");
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_service_info_creation() {
        let service = ServiceInfo::new("tli-service", true);
        assert_eq!(service.name, "tli-service");
        assert_eq!(service.container_name, "foxhunt-tli-service-test");
        assert!(service.requires_vault);
        assert_eq!(service.status, ServiceStatus::NotStarted);
    }

    #[test]
    fn test_health_url_mapping() {
        let tli_service = ServiceInfo::new("tli-service", true);
        assert!(tli_service.health_check_url.is_some());
        
        let unknown_service = ServiceInfo::new("unknown", false);
        assert!(unknown_service.health_check_url.is_none());
    }

    #[tokio::test]
    async fn test_service_integration_tester_creation() {
        let config = VaultTestConfig::default();
        let tester = ServiceIntegrationTester::new(config);
        
        assert_eq!(tester.services.len(), 2);
        assert!(tester.services.iter().all(|s| s.requires_vault));
    }
}