//! Enhanced Integration Testing Framework for Foxhunt HFT System //! //! This module provides a unified testing framework that orchestrates all three services //! (Trading, Backtesting, ML Training) along with TLI client testing, database hot-reload //! validation, and kill switch system verification. //! //! ## Key Features: //! - Unified service lifecycle management //! - Centralized mock implementations //! - Performance metrics collection //! - Cross-service integration validation //! - Kill switch emergency testing //! - Database hot-reload verification //! //! ## Usage: //! ```rust //! use tests::framework::TestOrchestrator; //! //! let orchestrator = TestOrchestrator::new().await?; //! orchestrator.run_integration_tests().await?; //! ``` pub mod orchestrator; pub mod mocks; pub mod metrics; pub mod services; use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::sync::{RwLock, broadcast, mpsc}; use tokio::time::timeout; use tracing::{info, warn, error, debug}; use uuid::Uuid; // use trading_engine::prelude::*; // REMOVED - prelude does not exist // use risk::prelude::*; // REMOVED - prelude does not exist /// Configuration settings for the test framework /// /// Defines timeouts, thresholds, and environment settings for /// comprehensive integration testing of the HFT system. #[derive(Debug, Clone)] pub struct TestFrameworkConfig { /// Maximum time allowed for any single test to execute pub max_test_timeout: Duration, /// Time to wait for services to start up before timing out pub service_startup_timeout: Duration, /// Time to wait for health check responses pub health_check_timeout: Duration, /// Database connection establishment timeout pub database_timeout: Duration, /// Maximum time for kill switch activation pub kill_switch_timeout: Duration, /// Performance thresholds for HFT requirements validation pub performance_thresholds: PerformanceThresholds, /// Target environment for test execution pub test_environment: TestEnvironment, } impl Default for TestFrameworkConfig { fn default() -> Self { Self { max_test_timeout: Duration::from_secs(300), service_startup_timeout: Duration::from_secs(30), health_check_timeout: Duration::from_secs(10), database_timeout: Duration::from_secs(15), kill_switch_timeout: Duration::from_secs(5), performance_thresholds: PerformanceThresholds::hft_defaults(), test_environment: TestEnvironment::Development, } } } /// Performance thresholds for HFT system validation /// /// Defines the maximum acceptable latencies and minimum throughput /// requirements for high-frequency trading operations. #[derive(Debug, Clone)] pub struct PerformanceThresholds { /// Maximum acceptable end-to-end trading latency in microseconds pub max_e2e_latency_us: u64, /// Maximum order processing latency in microseconds pub max_order_latency_us: u64, /// Maximum risk validation check latency in microseconds pub max_risk_latency_us: u64, /// Maximum ML model inference latency in milliseconds pub max_ml_latency_ms: u64, /// Maximum configuration hot-reload latency in milliseconds pub max_config_reload_ms: u64, /// Minimum required system throughput in operations per second pub min_throughput_ops_sec: u64, } impl PerformanceThresholds { /// Create performance thresholds suitable for high-frequency trading /// /// These defaults represent aggressive HFT requirements: /// - Sub-microsecond order processing /// - Ultra-low risk validation latency /// - High throughput requirements pub fn hft_defaults() -> Self { Self { max_e2e_latency_us: 50, // 50μs end-to-end max_order_latency_us: 20, // 20μs order processing max_risk_latency_us: 10, // 10μs risk validation max_ml_latency_ms: 50, // 50ms ML inference max_config_reload_ms: 100, // 100ms config reload min_throughput_ops_sec: 10000, // 10k ops/sec minimum } } } /// Different test environment configurations /// /// Each environment has different tolerance levels and requirements /// for performance, timeouts, and validation strictness. #[derive(Debug, Clone, PartialEq)] pub enum TestEnvironment { /// Local development environment with relaxed constraints Development, /// Continuous integration environment for automated testing CI, /// Staging environment that mirrors production Staging, /// Performance testing environment with strict HFT requirements Performance, } /// Complete result from an integration test execution /// /// Contains success status, timing information, performance metrics, /// and any errors or warnings encountered during the test. #[derive(Debug, Clone)] pub struct IntegrationTestResult { /// Name identifying the test that was executed pub test_name: String, /// Whether the test passed all validations pub success: bool, /// Total time taken to execute the test pub duration: Duration, /// Detailed performance and timing metrics pub metrics: TestMetrics, /// List of errors encountered (empty if test passed) pub errors: Vec, /// List of warnings (test may still pass with warnings) pub warnings: Vec, } /// Detailed metrics collected during test execution /// /// Captures performance data including service startup times, /// communication latencies, resource usage, and throughput measurements. #[derive(Debug, Clone, Default)] pub struct TestMetrics { /// Time taken for each service to start up (service name -> duration) pub service_startup_times: HashMap, /// gRPC call latencies by operation type (operation -> latencies) pub grpc_latencies: HashMap>, /// Database operation response times pub database_latencies: Vec, /// Time taken for kill switch activations pub kill_switch_times: Vec, /// Memory usage samples in bytes pub memory_usage: Vec, /// System throughput measurements in operations per second pub throughput_measurements: Vec, } /// Errors that can occur during test framework execution /// /// Covers all failure modes including service issues, performance /// threshold violations, and emergency procedure failures. #[derive(Debug, thiserror::Error)] pub enum TestFrameworkError { /// A service failed to start within the configured timeout #[error("Service startup timeout: {service}")] ServiceStartupTimeout { service: String }, /// Service health check endpoint is not responding properly #[error("Health check failed for service: {service}")] HealthCheckFailed { service: String }, /// A measured performance metric exceeded the HFT threshold #[error("Performance threshold exceeded: {metric} = {value:?}, limit = {limit:?}")] PerformanceThresholdExceeded { /// Name of the performance metric that failed metric: String, /// Actual measured value value: Duration, /// Maximum allowed value limit: Duration, }, /// Emergency kill switch failed to activate properly #[error("Kill switch activation failed: {reason}")] KillSwitchFailed { reason: String }, /// Database configuration hot-reload mechanism failed #[error("Database hot-reload failed: {reason}")] DatabaseHotReloadFailed { reason: String }, /// Communication or integration between services failed #[error("Cross-service integration failed: {reason}")] CrossServiceIntegrationFailed { reason: String }, /// A test exceeded its maximum allowed execution time #[error("Test timeout exceeded: {test_name}")] TestTimeout { test_name: String }, } pub type TestResult = std::result::Result;