foxhunt/tests/integration/config_hot_reload.rs

//! Configuration Hot-Reload Integration Tests
//!
//! This module provides comprehensive integration tests for configuration hot-reload
//! capabilities within the Foxhunt HFT system, including dynamic parameter updates,
//! rollback mechanisms, and configuration validation.
#![allow(unused_crate_dependencies)]

use std::collections::HashMap;
use std::sync::{Arc, atomic::{AtomicU64, AtomicBool, Ordering}};
use std::time::{Duration, Instant};
use std::path::{Path, PathBuf};

use tokio::sync::{RwLock, Mutex};
use tokio::fs;
use uuid::Uuid;
use serde_json::json;
use chrono::{DateTime, Utc};

use tli::prelude::*;
use crate::fixtures::{IntegrationTestConfig, TestEnvironment, TestMetricsCollector};
use crate::mocks::{MockTradingService, TestDatabaseManager};

/// Configuration hot-reload integration tests
pub struct ConfigHotReloadTests {
    client_suite: TliClientSuite,
    mock_trading_service: MockTradingService,
    test_db: TestDatabaseManager,
    config_manager: Arc<ConfigManager>,
    hot_reload_manager: Arc<HotReloadManager>,
    metrics: Arc<HotReloadMetrics>,
    config: IntegrationTestConfig,
    test_config_dir: PathBuf,
    config_files: Arc<RwLock<HashMap<String, PathBuf>>>,
}

/// Hot-reload performance metrics
#[derive(Debug, Default)]
pub struct HotReloadMetrics {
    pub config_reload_latency: AtomicU64,
    pub validation_latency: AtomicU64,
    pub rollback_latency: AtomicU64,
    pub configs_reloaded: AtomicU64,
    pub validation_failures: AtomicU64,
    pub rollbacks_performed: AtomicU64,
    pub notification_latency: AtomicU64,
    pub service_restart_count: AtomicU64,
}

impl HotReloadMetrics {
    pub fn new() -> Self {
        Self::default()
    }

    pub fn record_reload_latency(&self, latency_ns: u64) {
        self.config_reload_latency.store(latency_ns, Ordering::Relaxed);
        self.configs_reloaded.fetch_add(1, Ordering::Relaxed);
    }

    pub fn record_validation_latency(&self, latency_ns: u64) {
        self.validation_latency.store(latency_ns, Ordering::Relaxed);
    }

    pub fn record_validation_failure(&self) {
        self.validation_failures.fetch_add(1, Ordering::Relaxed);
    }

    pub fn record_rollback(&self, latency_ns: u64) {
        self.rollback_latency.store(latency_ns, Ordering::Relaxed);
        self.rollbacks_performed.fetch_add(1, Ordering::Relaxed);
    }

    pub fn record_notification_latency(&self, latency_ns: u64) {
        self.notification_latency.store(latency_ns, Ordering::Relaxed);
    }

    pub fn record_service_restart(&self) {
        self.service_restart_count.fetch_add(1, Ordering::Relaxed);
    }

    pub fn get_summary(&self) -> serde_json::Value {
        json!({
            "config_reload_latency_ns": self.config_reload_latency.load(Ordering::Relaxed),
            "validation_latency_ns": self.validation_latency.load(Ordering::Relaxed),
            "rollback_latency_ns": self.rollback_latency.load(Ordering::Relaxed),
            "configs_reloaded": self.configs_reloaded.load(Ordering::Relaxed),
            "validation_failures": self.validation_failures.load(Ordering::Relaxed),
            "rollbacks_performed": self.rollbacks_performed.load(Ordering::Relaxed),
            "notification_latency_ns": self.notification_latency.load(Ordering::Relaxed),
            "service_restarts": self.service_restart_count.load(Ordering::Relaxed)
        })
    }
}

impl ConfigHotReloadTests {
    /// Create new configuration hot-reload tests instance
    pub async fn new(config: IntegrationTestConfig) -> TliResult<Self> {
        let test_env = TestEnvironment::new(config.clone()).await?;

        // Initialize mock services
        let mock_trading_service = MockTradingService::new().await?;
        let test_db = TestDatabaseManager::new(&config.test_db_url).await?;

        // Create test configuration directory
        let test_config_dir = std::env::temp_dir().join(format!("config_test_{}", Uuid::new_v4()));
        fs::create_dir_all(&test_config_dir).await
            .map_err(|e| TliError::Other(format!("Failed to create test config dir: {}", e)))?;

        // Initialize configuration manager
        let config_manager_config = ConfigManagerConfig {
            database_url: config.test_db_url.clone(),
            encryption_key: Some("test_encryption_key_32_bytes_long".to_string()),
            cache_ttl_seconds: 300,
            batch_size: 100,
            max_connections: 10,
        };

        let config_manager = Arc::new(
            ConfigManager::new(config_manager_config).await
                .map_err(|e| TliError::Other(format!("Failed to create config manager: {}", e)))?
        );

        // Initialize hot-reload manager
        let hot_reload_config = HotReloadConfig {
            config_directory: test_config_dir.clone(),
            watch_patterns: vec!["*.json".to_string(), "*.toml".to_string(), "*.yaml".to_string()],
            debounce_duration: Duration::from_millis(100),
            validation_timeout: Duration::from_secs(5),
            rollback_on_validation_failure: true,
            max_rollback_history: 10,
            notification_channels: vec!["config_updates".to_string()],
        };

        let hot_reload_manager = Arc::new(
            HotReloadManager::new(hot_reload_config, Arc::clone(&config_manager)).await
                .map_err(|e| TliError::Other(format!("Failed to create hot-reload manager: {}", e)))?
        );

        // Create TLI client suite
        let client_suite = TliClientBuilder::new()
            .with_service_endpoint(
                "trading_service".to_string(),
                format!("http://localhost:{}", mock_trading_service.port())
            )
            .with_trading_config(TradingClientConfig::default())
            .build()
            .await?;

        Ok(Self {
            client_suite,
            mock_trading_service,
            test_db,
            config_manager,
            hot_reload_manager,
            metrics: Arc::new(HotReloadMetrics::new()),
            config,
            test_config_dir,
            config_files: Arc::new(RwLock::new(HashMap::new())),
        })
    }

    /// Test basic configuration hot-reload functionality
    pub async fn test_basic_config_reload(&mut self) -> TliResult<TestResult> {
        let mut test_result = TestResult::new("basic_config_reload");
        let start_time = Instant::now();

        println!("🔄 Testing basic configuration hot-reload...");

        // Create initial configuration file
        let config_file = self.test_config_dir.join("trading_params.json");
        let initial_config = json!({
            "max_position_size": 10000.0,
            "risk_multiplier": 1.5,
            "order_timeout_ms": 5000,
            "enable_stop_loss": true,
            "max_slippage_bps": 10
        });

        self.write_config_file(&config_file, &initial_config).await?;

        // Register configuration file with hot-reload manager
        self.hot_reload_manager.add_config_file("trading_params", &config_file).await
            .map_err(|e| TliError::Other(format!("Failed to register config file: {}", e)))?;

        // Setup change notification listener
        let notification_receiver = self.hot_reload_manager.subscribe_to_changes().await
            .map_err(|e| TliError::Other(format!("Failed to subscribe to changes: {}", e)))?;

        // Modify configuration file
        let reload_start = Instant::now();
        let updated_config = json!({
            "max_position_size": 15000.0,  // Changed
            "risk_multiplier": 2.0,        // Changed
            "order_timeout_ms": 3000,      // Changed
            "enable_stop_loss": true,
            "max_slippage_bps": 15         // Changed
        });

        self.write_config_file(&config_file, &updated_config).await?;

        // Wait for hot-reload notification
        let notification_timeout = Duration::from_secs(10);
        let mut reload_detected = false;
        let mut validation_passed = false;

        tokio::select! {
            result = tokio::time::timeout(notification_timeout, notification_receiver.recv()) => {
                match result {
                    Ok(Some(notification)) => {
                        let reload_latency = reload_start.elapsed().as_nanos() as u64;
                        self.metrics.record_reload_latency(reload_latency);
                        self.metrics.record_notification_latency(reload_latency);

                        reload_detected = true;

                        // Check if notification contains expected changes
                        if let Some(changes) = notification.changes {
                            validation_passed = changes.len() > 0;
                        }

                        println!("✅ Configuration reload detected in {}µs", reload_latency / 1000);
                    }
                    Ok(None) => {
                        test_result.add_error("Notification channel closed unexpectedly".to_string());
                    }
                    Err(_) => {
                        test_result.add_error("Timeout waiting for reload notification".to_string());
                    }
                }
            }
            _ = tokio::time::sleep(notification_timeout) => {
                test_result.add_error("No reload notification received within timeout".to_string());
            }
        }

        // Verify configuration was actually updated
        let current_config = self.config_manager.get_config("trading_params").await
            .map_err(|e| TliError::Other(format!("Failed to get current config: {}", e)))?;

        let config_updated = if let Some(config_value) = current_config {
            config_value.get("max_position_size")
                .and_then(|v| v.as_f64())
                .map(|v| v == 15000.0)
                .unwrap_or(false)
        } else {
            false
        };

        // Performance assertions
        let reload_latency = self.metrics.config_reload_latency.load(Ordering::Relaxed);
        test_result.add_assertion(
            &format!("Reload latency < 100ms (got {}µs)", reload_latency / 1000),
            reload_latency < 100_000_000 // 100ms in nanoseconds
        );

        test_result.add_assertion(
            "Configuration reload detected",
            reload_detected
        );

        test_result.add_assertion(
            "Configuration validation passed",
            validation_passed
        );

        test_result.add_assertion(
            "Configuration values updated correctly",
            config_updated
        );

        test_result.set_passed(test_result.assertions.iter().all(|a| a.passed));
        test_result.execution_time = start_time.elapsed();

        println!("✅ Basic config reload test completed");

        Ok(test_result)
    }

    /// Test configuration validation and rollback mechanisms
    pub async fn test_config_validation_rollback(&mut self) -> TliResult<TestResult> {
        let mut test_result = TestResult::new("config_validation_rollback");
        let start_time = Instant::now();

        println!("🔄 Testing configuration validation and rollback...");

        // Create initial valid configuration
        let config_file = self.test_config_dir.join("risk_params.json");
        let valid_config = json!({
            "max_drawdown_pct": 5.0,
            "position_limit": 1000000.0,
            "leverage_limit": 10.0,
            "var_confidence": 0.95
        });

        self.write_config_file(&config_file, &valid_config).await?;
        self.hot_reload_manager.add_config_file("risk_params", &config_file).await
            .map_err(|e| TliError::Other(format!("Failed to register config: {}", e)))?;

        // Setup validation rules
        let validation_rules = vec![
            ValidationRule {
                field: "max_drawdown_pct".to_string(),
                rule_type: "range".to_string(),
                parameters: json!({"min": 0.0, "max": 20.0}),
            },
            ValidationRule {
                field: "position_limit".to_string(),
                rule_type: "range".to_string(),
                parameters: json!({"min": 1000.0, "max": 10000000.0}),
            },
            ValidationRule {
                field: "leverage_limit".to_string(),
                rule_type: "range".to_string(),
                parameters: json!({"min": 1.0, "max": 50.0}),
            },
        ];

        for rule in validation_rules {
            self.hot_reload_manager.add_validation_rule("risk_params", rule).await
                .map_err(|e| TliError::Other(format!("Failed to add validation rule: {}", e)))?;
        }

        // Subscribe to rollback events
        let rollback_receiver = self.hot_reload_manager.subscribe_to_rollbacks().await
            .map_err(|e| TliError::Other(format!("Failed to subscribe to rollbacks: {}", e)))?;

        // Write invalid configuration (should trigger rollback)
        let rollback_start = Instant::now();
        let invalid_config = json!({
            "max_drawdown_pct": 25.0,      // Invalid: exceeds max 20.0
            "position_limit": 500.0,       // Invalid: below min 1000.0
            "leverage_limit": 100.0,       // Invalid: exceeds max 50.0
            "var_confidence": 0.95
        });

        self.write_config_file(&config_file, &invalid_config).await?;

        // Wait for validation failure and rollback
        let mut rollback_occurred = false;
        let mut validation_errors = Vec::new();

        tokio::select! {
            result = tokio::time::timeout(Duration::from_secs(10), rollback_receiver.recv()) => {
                match result {
                    Ok(Some(rollback_event)) => {
                        let rollback_latency = rollback_start.elapsed().as_nanos() as u64;
                        self.metrics.record_rollback(rollback_latency);
                        self.metrics.record_validation_failure();

                        rollback_occurred = true;
                        validation_errors = rollback_event.validation_errors;

                        println!("✅ Rollback completed in {}µs", rollback_latency / 1000);
                    }
                    Ok(None) => {
                        test_result.add_error("Rollback channel closed unexpectedly".to_string());
                    }
                    Err(_) => {
                        test_result.add_error("Timeout waiting for rollback".to_string());
                    }
                }
            }
            _ = tokio::time::sleep(Duration::from_secs(10)) => {
                test_result.add_error("No rollback occurred within timeout".to_string());
            }
        }

        // Verify configuration was rolled back to valid state
        let current_config = self.config_manager.get_config("risk_params").await
            .map_err(|e| TliError::Other(format!("Failed to get current config: {}", e)))?;

        let config_rolled_back = if let Some(config_value) = current_config {
            config_value.get("max_drawdown_pct")
                .and_then(|v| v.as_f64())
                .map(|v| v == 5.0) // Should be back to original value
                .unwrap_or(false)
        } else {
            false
        };

        // Performance and correctness assertions
        let rollback_latency = self.metrics.rollback_latency.load(Ordering::Relaxed);
        test_result.add_assertion(
            &format!("Rollback latency < 1s (got {}ms)", rollback_latency / 1_000_000),
            rollback_latency < 1_000_000_000 // 1s in nanoseconds
        );

        test_result.add_assertion(
            "Validation failure detected and rollback occurred",
            rollback_occurred
        );

        test_result.add_assertion(
            &format!("Validation errors reported (count: {})", validation_errors.len()),
            !validation_errors.is_empty()
        );

        test_result.add_assertion(
            "Configuration rolled back to valid state",
            config_rolled_back
        );

        test_result.set_passed(test_result.assertions.iter().all(|a| a.passed));
        test_result.execution_time = start_time.elapsed();

        println!("✅ Config validation and rollback test completed");

        Ok(test_result)
    }

    /// Test concurrent configuration changes
    pub async fn test_concurrent_config_changes(&mut self) -> TliResult<TestResult> {
        let mut test_result = TestResult::new("concurrent_config_changes");
        let start_time = Instant::now();

        println!("🔄 Testing concurrent configuration changes...");

        // Create multiple configuration files
        let config_files = vec![
            ("trading_params", "trading.json"),
            ("risk_params", "risk.json"),
            ("ml_params", "ml.json"),
            ("market_data_params", "market_data.json"),
        ];

        let mut file_paths = Vec::new();
        for (config_name, file_name) in &config_files {
            let config_file = self.test_config_dir.join(file_name);
            let initial_config = json!({
                "param1": 100.0,
                "param2": "initial_value",
                "param3": true,
                "timestamp": Utc::now().timestamp()
            });

            self.write_config_file(&config_file, &initial_config).await?;
            self.hot_reload_manager.add_config_file(config_name, &config_file).await
                .map_err(|e| TliError::Other(format!("Failed to register {}: {}", config_name, e)))?;

            file_paths.push((config_name.to_string(), config_file));
        }

        // Setup change notification listener
        let notification_receiver = self.hot_reload_manager.subscribe_to_changes().await
            .map_err(|e| TliError::Other(format!("Failed to subscribe to changes: {}", e)))?;

        // Concurrently modify all configuration files
        let concurrent_start = Instant::now();
        let mut change_tasks = Vec::new();

        for (i, (config_name, config_file)) in file_paths.into_iter().enumerate() {
            let config_file_clone = config_file.clone();
            let config_name_clone = config_name.clone();

            let task = tokio::spawn(async move {
                // Stagger changes slightly to test race conditions
                tokio::time::sleep(Duration::from_millis(i as u64 * 10)).await;

                let updated_config = json!({
                    "param1": 200.0 + (i as f64 * 10.0),
                    "param2": format!("updated_value_{}", i),
                    "param3": i % 2 == 0,
                    "timestamp": Utc::now().timestamp(),
                    "config_id": config_name_clone
                });

                tokio::fs::write(&config_file_clone, serde_json::to_string_pretty(&updated_config).unwrap()).await
            });

            change_tasks.push(task);
        }

        // Wait for all changes to be written
        let write_results: Vec<_> = futures::future::join_all(change_tasks).await;
        let write_errors: Vec<_> = write_results.into_iter().filter_map(|r| r.err()).collect();

        if !write_errors.is_empty() {
            test_result.add_error(format!("Failed to write some config files: {:?}", write_errors));
        }

        // Collect reload notifications
        let mut notifications_received = 0;
        let mut reload_latencies = Vec::new();
        let notification_timeout = Duration::from_secs(15);
        let collection_deadline = Instant::now() + notification_timeout;

        while Instant::now() < collection_deadline && notifications_received < config_files.len() {
            tokio::select! {
                result = notification_receiver.recv() => {
                    match result {
                        Some(notification) => {
                            let latency = concurrent_start.elapsed().as_nanos() as u64;
                            reload_latencies.push(latency);
                            notifications_received += 1;

                            self.metrics.record_reload_latency(latency);
                            println!("📨 Received notification {} for config changes", notifications_received);
                        }
                        None => {
                            test_result.add_error("Notification channel closed".to_string());
                            break;
                        }
                    }
                }
                _ = tokio::time::sleep(Duration::from_millis(100)) => {
                    // Continue waiting
                }
            }
        }

        // Verify all configurations were updated
        let mut configs_updated = 0;
        for (config_name, _) in &config_files {
            if let Ok(Some(config)) = self.config_manager.get_config(config_name).await {
                if config.get("param1")
                    .and_then(|v| v.as_f64())
                    .map(|v| v >= 200.0)
                    .unwrap_or(false) {
                    configs_updated += 1;
                }
            }
        }

        // Calculate performance metrics
        if !reload_latencies.is_empty() {
            reload_latencies.sort();
            let avg_latency = reload_latencies.iter().sum::<u64>() / reload_latencies.len() as u64;
            let max_latency = reload_latencies.last().copied().unwrap_or(0);

            // Performance assertions
            test_result.add_assertion(
                &format!("All {} notifications received", config_files.len()),
                notifications_received == config_files.len()
            );

            test_result.add_assertion(
                &format!("Average reload latency < 500ms (got {}ms)", avg_latency / 1_000_000),
                avg_latency < 500_000_000 // 500ms in nanoseconds
            );

            test_result.add_assertion(
                &format!("Max reload latency < 2s (got {}ms)", max_latency / 1_000_000),
                max_latency < 2_000_000_000 // 2s in nanoseconds
            );
        }

        test_result.add_assertion(
            &format!("All {} configurations updated correctly", config_files.len()),
            configs_updated == config_files.len()
        );

        test_result.add_assertion(
            "No race conditions detected",
            write_errors.is_empty()
        );

        test_result.set_passed(test_result.assertions.iter().all(|a| a.passed));
        test_result.execution_time = start_time.elapsed();

        // Store concurrent test metadata
        test_result.metadata.insert("concurrent_configs".to_string(), json!(config_files.len()));
        test_result.metadata.insert("notifications_received".to_string(), json!(notifications_received));
        test_result.metadata.insert("configs_updated".to_string(), json!(configs_updated));

        println!("✅ Concurrent config changes test completed: {}/{} configs updated",
                configs_updated, config_files.len());

        Ok(test_result)
    }

    /// Test configuration change impact on running services
    pub async fn test_service_integration(&mut self) -> TliResult<TestResult> {
        let mut test_result = TestResult::new("service_integration");
        let start_time = Instant::now();

        println!("🔄 Testing configuration change impact on running services...");

        // Create service configuration file
        let service_config_file = self.test_config_dir.join("service_params.json");
        let initial_service_config = json!({
            "max_concurrent_orders": 100,
            "order_queue_size": 1000,
            "heartbeat_interval_ms": 1000,
            "request_timeout_ms": 5000,
            "circuit_breaker_threshold": 5
        });

        self.write_config_file(&service_config_file, &initial_service_config).await?;
        self.hot_reload_manager.add_config_file("service_params", &service_config_file).await
            .map_err(|e| TliError::Other(format!("Failed to register service config: {}", e)))?;

        // Configure mock trading service to respond to config changes
        self.mock_trading_service.set_config_update_handler(Box::new(|config| {
            println!("🔧 Trading service received config update: {:?}", config);
            // Simulate service reconfiguration time
            std::thread::sleep(Duration::from_millis(50));
            Ok(())
        })).await?;

        // Subscribe to service notifications
        let service_receiver = self.hot_reload_manager.subscribe_to_service_updates().await
            .map_err(|e| TliError::Other(format!("Failed to subscribe to service updates: {}", e)))?;

        // Submit some orders before configuration change
        let initial_orders = 10;
        for i in 0..initial_orders {
            let order_request = SubmitOrderRequest {
                symbol: "INTEGRATION_TEST".to_string(),
                side: OrderSide::Buy as i32,
                order_type: OrderType::Market as i32,
                quantity: 100.0,
                client_order_id: format!("pre_config_order_{}", i),
                ..Default::default()
            };

            if let Some(trading_client) = &self.client_suite.trading_client {
                let _ = trading_client.submit_order(order_request).await;
            }
        }

        // Update service configuration
        let service_update_start = Instant::now();
        let updated_service_config = json!({
            "max_concurrent_orders": 200,     // Doubled
            "order_queue_size": 2000,         // Doubled
            "heartbeat_interval_ms": 500,     // Halved
            "request_timeout_ms": 3000,       // Reduced
            "circuit_breaker_threshold": 10   // Increased
        });

        self.write_config_file(&service_config_file, &updated_service_config).await?;

        // Wait for service update notification
        let mut service_updated = false;
        let mut service_restart_required = false;

        tokio::select! {
            result = tokio::time::timeout(Duration::from_secs(10), service_receiver.recv()) => {
                match result {
                    Ok(Some(update_event)) => {
                        let update_latency = service_update_start.elapsed().as_nanos() as u64;
                        self.metrics.record_notification_latency(update_latency);

                        service_updated = true;
                        service_restart_required = update_event.requires_restart;

                        if service_restart_required {
                            self.metrics.record_service_restart();
                        }

                        println!("🔄 Service update processed in {}µs (restart required: {})",
                                update_latency / 1000, service_restart_required);
                    }
                    Ok(None) => {
                        test_result.add_error("Service update channel closed".to_string());
                    }
                    Err(_) => {
                        test_result.add_error("Timeout waiting for service update".to_string());
                    }
                }
            }
            _ = tokio::time::sleep(Duration::from_secs(10)) => {
                test_result.add_error("No service update notification received".to_string());
            }
        }

        // Test that service continues to function after configuration update
        let post_config_orders = 5;
        let mut successful_post_config_orders = 0;

        for i in 0..post_config_orders {
            let order_request = SubmitOrderRequest {
                symbol: "POST_CONFIG_TEST".to_string(),
                side: OrderSide::Sell as i32,
                order_type: OrderType::Limit as i32,
                quantity: 50.0,
                price: Some(150.0 + i as f64),
                client_order_id: format!("post_config_order_{}", i),
                ..Default::default()
            };

            if let Some(trading_client) = &self.client_suite.trading_client {
                match trading_client.submit_order(order_request).await {
                    Ok(_) => successful_post_config_orders += 1,
                    Err(e) => {
                        test_result.add_error(format!("Post-config order {} failed: {}", i, e));
                    }
                }
            }
        }

        // Service integration assertions
        test_result.add_assertion(
            "Service configuration update detected",
            service_updated
        );

        test_result.add_assertion(
            &format!("Post-config orders successful: {}/{}", successful_post_config_orders, post_config_orders),
            successful_post_config_orders == post_config_orders
        );

        let notification_latency = self.metrics.notification_latency.load(Ordering::Relaxed);
        test_result.add_assertion(
            &format!("Service notification latency < 1s (got {}ms)", notification_latency / 1_000_000),
            notification_latency < 1_000_000_000 // 1s in nanoseconds
        );

        // Verify configuration was applied to the service
        let current_service_config = self.config_manager.get_config("service_params").await
            .map_err(|e| TliError::Other(format!("Failed to get service config: {}", e)))?;

        let config_applied = if let Some(config) = current_service_config {
            config.get("max_concurrent_orders")
                .and_then(|v| v.as_u64())
                .map(|v| v == 200)
                .unwrap_or(false)
        } else {
            false
        };

        test_result.add_assertion(
            "Updated configuration applied to service",
            config_applied
        );

        test_result.set_passed(test_result.assertions.iter().all(|a| a.passed));
        test_result.execution_time = start_time.elapsed();

        println!("✅ Service integration test completed: config updated and service functional");

        Ok(test_result)
    }

    /// Write configuration to file
    async fn write_config_file(&self, path: &Path, config: &serde_json::Value) -> TliResult<()> {
        let config_str = serde_json::to_string_pretty(config)
            .map_err(|e| TliError::Other(format!("Failed to serialize config: {}", e)))?;

        fs::write(path, config_str).await
            .map_err(|e| TliError::Other(format!("Failed to write config file: {}", e)))?;

        Ok(())
    }

    /// Run all configuration hot-reload integration tests
    pub async fn run_all_tests(&mut self) -> TliResult<TestSuite> {
        let mut test_suite = TestSuite::new("config_hot_reload_integration");
        println!("🚀 Starting configuration hot-reload integration tests...");

        // Run individual test methods
        let tests = vec![
            self.test_basic_config_reload().await,
            self.test_config_validation_rollback().await,
            self.test_concurrent_config_changes().await,
            self.test_service_integration().await,
        ];

        // Collect results
        for test_result in tests {
            match test_result {
                Ok(result) => {
                    test_suite.add_test_result(result);
                }
                Err(e) => {
                    let mut error_result = TestResult::new("config_reload_test_error");
                    error_result.add_error(format!("Test execution failed: {}", e));
                    test_suite.add_test_result(error_result);
                }
            }
        }

        // Calculate overall success
        test_suite.set_passed(test_suite.passed_tests == test_suite.total_tests);

        // Add hot-reload metrics to test suite metadata
        let metrics_summary = self.metrics.get_summary();
        test_suite.metadata.insert("hot_reload_metrics".to_string(), metrics_summary);

        println!("🏁 Configuration hot-reload integration tests completed: {}/{} passed",
                test_suite.passed_tests, test_suite.total_tests);

        Ok(test_suite)
    }

    /// Cleanup test resources
    pub async fn cleanup(&self) -> TliResult<()> {
        // Remove test configuration directory
        if self.test_config_dir.exists() {
            fs::remove_dir_all(&self.test_config_dir).await
                .map_err(|e| TliError::Other(format!("Failed to cleanup test config dir: {}", e)))?;
        }

        // Shutdown hot-reload manager
        self.hot_reload_manager.shutdown().await
            .map_err(|e| TliError::Other(format!("Failed to shutdown hot-reload manager: {}", e)))?;

        Ok(())
    }
}

/// Test result structure for configuration tests
#[derive(Debug, Clone)]
pub struct TestResult {
    pub name: String,
    pub passed: bool,
    pub execution_time: Duration,
    pub assertions: Vec<Assertion>,
    pub errors: Vec<String>,
    pub metadata: HashMap<String, serde_json::Value>,
}

impl TestResult {
    pub fn new(name: &str) -> Self {
        Self {
            name: name.to_string(),
            passed: false,
            execution_time: Duration::default(),
            assertions: Vec::new(),
            errors: Vec::new(),
            metadata: HashMap::new(),
        }
    }

    pub fn add_assertion(&mut self, description: &str, passed: bool) {
        self.assertions.push(Assertion {
            description: description.to_string(),
            passed,
        });
    }

    pub fn add_error(&mut self, error: String) {
        self.errors.push(error);
    }

    pub fn set_passed(&mut self, passed: bool) {
        self.passed = passed;
    }
}

/// Individual test assertion
#[derive(Debug, Clone)]
pub struct Assertion {
    pub description: String,
    pub passed: bool,
}

/// Test suite containing multiple configuration test results
#[derive(Debug, Clone)]
pub struct TestSuite {
    pub name: String,
    pub tests: Vec<TestResult>,
    pub passed_tests: usize,
    pub total_tests: usize,
    pub passed: bool,
    pub execution_time: Duration,
    pub metadata: HashMap<String, serde_json::Value>,
}

impl TestSuite {
    pub fn new(name: &str) -> Self {
        Self {
            name: name.to_string(),
            tests: Vec::new(),
            passed_tests: 0,
            total_tests: 0,
            passed: false,
            execution_time: Duration::default(),
            metadata: HashMap::new(),
        }
    }

    pub fn add_test_result(&mut self, test: TestResult) {
        if test.passed {
            self.passed_tests += 1;
        }
        self.total_tests += 1;
        self.tests.push(test);
    }

    pub fn set_passed(&mut self, passed: bool) {
        self.passed = passed;
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_hot_reload_metrics() {
        let metrics = HotReloadMetrics::new();

        metrics.record_reload_latency(50_000_000); // 50ms
        metrics.record_validation_latency(10_000_000); // 10ms
        metrics.record_rollback(75_000_000); // 75ms

        let summary = metrics.get_summary();
        assert_eq!(summary["config_reload_latency_ns"].as_u64().unwrap(), 50_000_000);
        assert_eq!(summary["configs_reloaded"].as_u64().unwrap(), 1);
        assert_eq!(summary["rollbacks_performed"].as_u64().unwrap(), 1);
    }
}