//! Comprehensive health check tests for Backtesting Service //! //! Tests cover: //! - HTTP /health and /ready endpoints //! - Service startup state transitions //! - Database connectivity checks //! - Storage backend availability //! - Data replay capability //! - Dependency cascade failures //! - Health check latency //! - Concurrent health checks //! - Health during backtest execution //! - Resource exhaustion scenarios use axum::{ body::Body, http::{Request, StatusCode}, }; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::sync::RwLock; use tower::ServiceExt; /// Mock health state for backtesting service #[derive(Clone)] struct MockBacktestHealthState { healthy: Arc>, ready: Arc>, database_connected: Arc>, storage_available: Arc>, backtest_running: Arc>, } impl MockBacktestHealthState { fn new() -> Self { Self { healthy: Arc::new(RwLock::new(true)), ready: Arc::new(RwLock::new(true)), database_connected: Arc::new(RwLock::new(true)), storage_available: Arc::new(RwLock::new(true)), backtest_running: Arc::new(RwLock::new(false)), } } async fn set_healthy(&self, healthy: bool) { *self.healthy.write().await = healthy; } async fn set_ready(&self, ready: bool) { *self.ready.write().await = ready; } async fn set_database_connected(&self, connected: bool) { *self.database_connected.write().await = connected; } async fn set_storage_available(&self, available: bool) { *self.storage_available.write().await = available; } async fn set_backtest_running(&self, running: bool) { *self.backtest_running.write().await = running; } async fn is_healthy(&self) -> bool { *self.healthy.read().await } async fn is_ready(&self) -> bool { *self.ready.read().await } async fn is_database_connected(&self) -> bool { *self.database_connected.read().await } async fn is_storage_available(&self) -> bool { *self.storage_available.read().await } async fn is_backtest_running(&self) -> bool { *self.backtest_running.read().await } } /// Create health router for backtesting service fn create_backtest_health_router(state: MockBacktestHealthState) -> axum::Router { use axum::{extract::State, routing::get, Json, Router}; use serde_json::json; async fn health_handler( State(state): State, ) -> Result, StatusCode> { if state.is_healthy().await { Ok(Json(json!({ "status": "healthy", "service": "backtesting", "version": env!("CARGO_PKG_VERSION") }))) } else { Err(StatusCode::SERVICE_UNAVAILABLE) } } async fn ready_handler( State(state): State, ) -> Result, StatusCode> { if state.is_ready().await { Ok(Json(json!({ "status": "ready", "service": "backtesting", "version": env!("CARGO_PKG_VERSION") }))) } else { Err(StatusCode::SERVICE_UNAVAILABLE) } } async fn deep_health_handler( State(state): State, ) -> Result, StatusCode> { let db_ok = state.is_database_connected().await; let storage_ok = state.is_storage_available().await; let healthy = state.is_healthy().await; let backtest_running = state.is_backtest_running().await; if healthy && db_ok && storage_ok { Ok(Json(json!({ "status": "healthy", "service": "backtesting", "version": env!("CARGO_PKG_VERSION"), "dependencies": { "database": "ok", "storage": "ok" }, "backtest_running": backtest_running }))) } else { Err(StatusCode::SERVICE_UNAVAILABLE) } } Router::new() .route("/health", get(health_handler)) .route("/ready", get(ready_handler)) .route("/health/deep", get(deep_health_handler)) .with_state(state) } #[tokio::test] async fn test_backtest_health_basic_healthy() { let state = MockBacktestHealthState::new(); let app = create_backtest_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/health") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); } #[tokio::test] async fn test_backtest_health_unhealthy() { let state = MockBacktestHealthState::new(); state.set_healthy(false).await; let app = create_backtest_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/health") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); } #[tokio::test] async fn test_backtest_readiness_check() { let state = MockBacktestHealthState::new(); let app = create_backtest_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/ready") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); } #[tokio::test] async fn test_backtest_service_startup() { let state = MockBacktestHealthState::new(); state.set_ready(false).await; // Initially not ready assert!(!state.is_ready().await); assert!(state.is_healthy().await); // Simulate startup tokio::time::sleep(Duration::from_millis(50)).await; state.set_ready(true).await; assert!(state.is_ready().await); } #[tokio::test] async fn test_backtest_database_disconnection() { let state = MockBacktestHealthState::new(); state.set_database_connected(false).await; let app = create_backtest_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/health/deep") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); } #[tokio::test] async fn test_backtest_storage_unavailable() { let state = MockBacktestHealthState::new(); state.set_storage_available(false).await; let app = create_backtest_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/health/deep") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); } #[tokio::test] async fn test_backtest_health_during_execution() { let state = MockBacktestHealthState::new(); state.set_backtest_running(true).await; let app = create_backtest_health_router(state.clone()); // Service should still be healthy during backtest execution let response = app .oneshot( Request::builder() .uri("/health") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); } #[tokio::test] async fn test_backtest_dependency_cascade_failure() { let state = MockBacktestHealthState::new(); // Both dependencies fail state.set_database_connected(false).await; state.set_storage_available(false).await; let app = create_backtest_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/health/deep") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); } #[tokio::test] async fn test_backtest_health_check_latency() { let state = MockBacktestHealthState::new(); let app = create_backtest_health_router(state.clone()); let start = Instant::now(); let response = app .oneshot( Request::builder() .uri("/health") .body(Body::empty()) .unwrap(), ) .await .unwrap(); let latency = start.elapsed(); assert_eq!(response.status(), StatusCode::OK); assert!( latency < Duration::from_millis(100), "Health check latency: {:?}", latency ); } #[tokio::test] async fn test_backtest_concurrent_health_checks() { let state = MockBacktestHealthState::new(); let mut handles = vec![]; for _ in 0..100 { let state_clone = state.clone(); let handle = tokio::spawn(async move { let app = create_backtest_health_router(state_clone); let response = app .oneshot( Request::builder() .uri("/health") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); }); handles.push(handle); } for handle in handles { handle.await.unwrap(); } } #[tokio::test] async fn test_backtest_health_during_shutdown() { let state = MockBacktestHealthState::new(); // Graceful shutdown state.set_ready(false).await; state.set_healthy(true).await; let app = create_backtest_health_router(state.clone()); // Ready check fails let response = app .clone() .oneshot( Request::builder() .uri("/ready") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); // Health check passes let response = app .oneshot( Request::builder() .uri("/health") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); } #[tokio::test] async fn test_backtest_rapid_health_checks() { let state = MockBacktestHealthState::new(); let start = Instant::now(); for _ in 0..500 { let app = create_backtest_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/health") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); } let duration = start.elapsed(); assert!( duration < Duration::from_secs(1), "500 health checks took: {:?}", duration ); } #[tokio::test] async fn test_backtest_deep_vs_shallow_health() { let state = MockBacktestHealthState::new(); let app = create_backtest_health_router(state.clone()); // Shallow health let start = Instant::now(); let response = app .clone() .oneshot( Request::builder() .uri("/health") .body(Body::empty()) .unwrap(), ) .await .unwrap(); let shallow_latency = start.elapsed(); assert_eq!(response.status(), StatusCode::OK); // Deep health let start = Instant::now(); let response = app .oneshot( Request::builder() .uri("/health/deep") .body(Body::empty()) .unwrap(), ) .await .unwrap(); let deep_latency = start.elapsed(); assert_eq!(response.status(), StatusCode::OK); assert!(shallow_latency < Duration::from_millis(50)); assert!(deep_latency < Duration::from_millis(100)); } #[tokio::test] async fn test_backtest_partial_availability() { let state = MockBacktestHealthState::new(); // Database down, storage up state.set_database_connected(false).await; state.set_storage_available(true).await; let app = create_backtest_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/health/deep") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); } #[tokio::test] async fn test_backtest_recovery_after_failure() { let state = MockBacktestHealthState::new(); // Service fails state.set_healthy(false).await; let app = create_backtest_health_router(state.clone()); let response = app .clone() .oneshot( Request::builder() .uri("/health") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); // Service recovers state.set_healthy(true).await; let response = app .oneshot( Request::builder() .uri("/health") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); } #[tokio::test] async fn test_backtest_health_json_format() { let state = MockBacktestHealthState::new(); let app = create_backtest_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/health") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); let content_type = response.headers().get("content-type"); assert!(content_type.is_some()); let content_type_str = content_type.unwrap().to_str().expect("INVARIANT: Path should be valid UTF-8"); assert!(content_type_str.contains("application/json")); }