foxhunt/tests/framework/mod.rs

//! Enhanced Integration Testing Framework for Foxhunt HFT System
//! 
//! This module provides a unified testing framework that orchestrates all three services
//! (Trading, Backtesting, ML Training) along with TLI client testing, database hot-reload
//! validation, and kill switch system verification.
//!
//! ## Key Features:
//! - Unified service lifecycle management
//! - Centralized mock implementations
//! - Performance metrics collection
//! - Cross-service integration validation
//! - Kill switch emergency testing
//! - Database hot-reload verification
//! 
//! ## Usage:
//! ```rust
//! use tests::framework::TestOrchestrator;
//! 
//! let orchestrator = TestOrchestrator::new().await?;
//! orchestrator.run_integration_tests().await?;
//! ```

pub mod orchestrator;
pub mod mocks;
pub mod metrics;
pub mod services;


use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::{RwLock, broadcast, mpsc};
use tokio::time::timeout;
use tracing::{info, warn, error, debug};
use uuid::Uuid;

use trading_engine::prelude::*;
use risk::prelude::*;

/// Configuration settings for the test framework
/// 
/// Defines timeouts, thresholds, and environment settings for
/// comprehensive integration testing of the HFT system.
#[derive(Debug, Clone)]
pub struct TestFrameworkConfig {
    /// Maximum time allowed for any single test to execute
    pub max_test_timeout: Duration,
    /// Time to wait for services to start up before timing out
    pub service_startup_timeout: Duration,
    /// Time to wait for health check responses
    pub health_check_timeout: Duration,
    /// Database connection establishment timeout
    pub database_timeout: Duration,
    /// Maximum time for kill switch activation
    pub kill_switch_timeout: Duration,
    /// Performance thresholds for HFT requirements validation
    pub performance_thresholds: PerformanceThresholds,
    /// Target environment for test execution
    pub test_environment: TestEnvironment,
}

impl Default for TestFrameworkConfig {
    fn default() -> Self {
        Self {
            max_test_timeout: Duration::from_secs(300),
            service_startup_timeout: Duration::from_secs(30),
            health_check_timeout: Duration::from_secs(10),
            database_timeout: Duration::from_secs(15),
            kill_switch_timeout: Duration::from_secs(5),
            performance_thresholds: PerformanceThresholds::hft_defaults(),
            test_environment: TestEnvironment::Development,
        }
    }
}

/// Performance thresholds for HFT system validation
/// 
/// Defines the maximum acceptable latencies and minimum throughput
/// requirements for high-frequency trading operations.
#[derive(Debug, Clone)]
pub struct PerformanceThresholds {
    /// Maximum acceptable end-to-end trading latency in microseconds
    pub max_e2e_latency_us: u64,
    /// Maximum order processing latency in microseconds
    pub max_order_latency_us: u64,
    /// Maximum risk validation check latency in microseconds
    pub max_risk_latency_us: u64,
    /// Maximum ML model inference latency in milliseconds
    pub max_ml_latency_ms: u64,
    /// Maximum configuration hot-reload latency in milliseconds
    pub max_config_reload_ms: u64,
    /// Minimum required system throughput in operations per second
    pub min_throughput_ops_sec: u64,
}

impl PerformanceThresholds {
    /// Create performance thresholds suitable for high-frequency trading
    /// 
    /// These defaults represent aggressive HFT requirements:
    /// - Sub-microsecond order processing
    /// - Ultra-low risk validation latency
    /// - High throughput requirements
    pub fn hft_defaults() -> Self {
        Self {
            max_e2e_latency_us: 50,        // 50μs end-to-end
            max_order_latency_us: 20,      // 20μs order processing
            max_risk_latency_us: 10,       // 10μs risk validation
            max_ml_latency_ms: 50,         // 50ms ML inference
            max_config_reload_ms: 100,     // 100ms config reload
            min_throughput_ops_sec: 10000, // 10k ops/sec minimum
        }
    }
}

/// Different test environment configurations
/// 
/// Each environment has different tolerance levels and requirements
/// for performance, timeouts, and validation strictness.
#[derive(Debug, Clone, PartialEq)]
pub enum TestEnvironment {
    /// Local development environment with relaxed constraints
    Development,
    /// Continuous integration environment for automated testing
    CI,
    /// Staging environment that mirrors production
    Staging,
    /// Performance testing environment with strict HFT requirements
    Performance,
}

/// Complete result from an integration test execution
/// 
/// Contains success status, timing information, performance metrics,
/// and any errors or warnings encountered during the test.
#[derive(Debug, Clone)]
pub struct IntegrationTestResult {
    /// Name identifying the test that was executed
    pub test_name: String,
    /// Whether the test passed all validations
    pub success: bool,
    /// Total time taken to execute the test
    pub duration: Duration,
    /// Detailed performance and timing metrics
    pub metrics: TestMetrics,
    /// List of errors encountered (empty if test passed)
    pub errors: Vec<String>,
    /// List of warnings (test may still pass with warnings)
    pub warnings: Vec<String>,
}

/// Detailed metrics collected during test execution
/// 
/// Captures performance data including service startup times,
/// communication latencies, resource usage, and throughput measurements.
#[derive(Debug, Clone, Default)]
pub struct TestMetrics {
    /// Time taken for each service to start up (service name -> duration)
    pub service_startup_times: HashMap<String, Duration>,
    /// gRPC call latencies by operation type (operation -> latencies)
    pub grpc_latencies: HashMap<String, Vec<Duration>>,
    /// Database operation response times
    pub database_latencies: Vec<Duration>,
    /// Time taken for kill switch activations
    pub kill_switch_times: Vec<Duration>,
    /// Memory usage samples in bytes
    pub memory_usage: Vec<u64>,
    /// System throughput measurements in operations per second
    pub throughput_measurements: Vec<u64>,
}

/// Errors that can occur during test framework execution
/// 
/// Covers all failure modes including service issues, performance
/// threshold violations, and emergency procedure failures.
#[derive(Debug, thiserror::Error)]
pub enum TestFrameworkError {
    /// A service failed to start within the configured timeout
    #[error("Service startup timeout: {service}")]
    ServiceStartupTimeout { service: String },
    
    /// Service health check endpoint is not responding properly
    #[error("Health check failed for service: {service}")]
    HealthCheckFailed { service: String },
    
    /// A measured performance metric exceeded the HFT threshold
    #[error("Performance threshold exceeded: {metric} = {value:?}, limit = {limit:?}")]
    PerformanceThresholdExceeded {
        /// Name of the performance metric that failed
        metric: String,
        /// Actual measured value
        value: Duration,
        /// Maximum allowed value
        limit: Duration,
    },
    
    /// Emergency kill switch failed to activate properly
    #[error("Kill switch activation failed: {reason}")]
    KillSwitchFailed { reason: String },
    
    /// Database configuration hot-reload mechanism failed
    #[error("Database hot-reload failed: {reason}")]
    DatabaseHotReloadFailed { reason: String },
    
    /// Communication or integration between services failed
    #[error("Cross-service integration failed: {reason}")]
    CrossServiceIntegrationFailed { reason: String },
    
    /// A test exceeded its maximum allowed execution time
    #[error("Test timeout exceeded: {test_name}")]
    TestTimeout { test_name: String },
}

pub type TestResult<T> = std::result::Result<T, TestFrameworkError>;