//! Comprehensive health check tests for API Gateway //! //! Tests cover: //! - HTTP /health/liveness, /health/readiness, /health/startup endpoints //! - Backend service health checks //! - Rate limiter status //! - Circuit breaker status //! - Service startup transitions //! - Backend service cascade failures //! - Health check latency //! - Concurrent health checks //! - Health during high load //! - Resilience endpoint tests use axum::{ body::Body, http::{Request, StatusCode}, }; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::sync::RwLock; use tower::ServiceExt; /// Mock health state for API Gateway #[derive(Clone)] struct MockGatewayHealthState { startup_complete: Arc>, service_healthy: Arc>, trading_service_up: Arc>, backtesting_service_up: Arc>, ml_service_up: Arc>, rate_limiter_healthy: Arc>, circuit_breaker_open: Arc>, } impl MockGatewayHealthState { fn new() -> Self { Self { startup_complete: Arc::new(RwLock::new(true)), service_healthy: Arc::new(RwLock::new(true)), trading_service_up: Arc::new(RwLock::new(true)), backtesting_service_up: Arc::new(RwLock::new(true)), ml_service_up: Arc::new(RwLock::new(true)), rate_limiter_healthy: Arc::new(RwLock::new(true)), circuit_breaker_open: Arc::new(RwLock::new(false)), } } async fn mark_startup_complete(&self) { *self.startup_complete.write().await = true; } async fn set_healthy(&self, healthy: bool) { *self.service_healthy.write().await = healthy; } async fn set_trading_service_up(&self, up: bool) { *self.trading_service_up.write().await = up; } async fn set_backtesting_service_up(&self, up: bool) { *self.backtesting_service_up.write().await = up; } async fn set_ml_service_up(&self, up: bool) { *self.ml_service_up.write().await = up; } async fn set_rate_limiter_healthy(&self, healthy: bool) { *self.rate_limiter_healthy.write().await = healthy; } async fn set_circuit_breaker_open(&self, open: bool) { *self.circuit_breaker_open.write().await = open; } async fn is_startup_complete(&self) -> bool { *self.startup_complete.read().await } async fn is_healthy(&self) -> bool { *self.service_healthy.read().await } async fn is_trading_service_up(&self) -> bool { *self.trading_service_up.read().await } async fn is_backtesting_service_up(&self) -> bool { *self.backtesting_service_up.read().await } async fn is_ml_service_up(&self) -> bool { *self.ml_service_up.read().await } async fn is_rate_limiter_healthy(&self) -> bool { *self.rate_limiter_healthy.read().await } async fn is_circuit_breaker_open(&self) -> bool { *self.circuit_breaker_open.read().await } } /// Create health router for API Gateway fn create_gateway_health_router(state: MockGatewayHealthState) -> axum::Router { use axum::{extract::State, routing::get, Json, Router}; use serde_json::json; async fn liveness_handler() -> &'static str { "OK" } async fn readiness_handler( State(state): State, ) -> Result { if state.is_healthy().await { Ok("READY".to_string()) } else { Err(StatusCode::SERVICE_UNAVAILABLE) } } async fn startup_handler( State(state): State, ) -> Result { if state.is_startup_complete().await { Ok("READY".to_string()) } else { Err(StatusCode::SERVICE_UNAVAILABLE) } } async fn circuit_breaker_status_handler( State(state): State, ) -> Json { let is_open = state.is_circuit_breaker_open().await; Json(json!({ "state": if is_open { "open" } else { "closed" }, "failure_count": if is_open { 5 } else { 0 }, "success_count": 100, "threshold": 5, "timeout_seconds": 60 })) } async fn rate_limit_status_handler( State(state): State, ) -> Json { let healthy = state.is_rate_limiter_healthy().await; Json(json!({ "enabled": true, "requests_per_minute": 1000, "current_usage": if healthy { 0 } else { 1000 }, "burst_size": 100 })) } async fn timeout_config_handler() -> Json { Json(json!({ "request_timeout_ms": 5000, "connection_timeout_ms": 3000, "idle_timeout_ms": 60000 })) } async fn retry_config_handler() -> Json { Json(json!({ "max_retries": 3, "retry_delay_ms": 100, "backoff_multiplier": 2.0, "max_backoff_ms": 10000 })) } async fn backend_services_handler( State(state): State, ) -> Json { let trading_up = state.is_trading_service_up().await; let backtesting_up = state.is_backtesting_service_up().await; let ml_up = state.is_ml_service_up().await; Json(json!({ "trading_service": if trading_up { "up" } else { "down" }, "backtesting_service": if backtesting_up { "up" } else { "down" }, "ml_training_service": if ml_up { "up" } else { "down" } })) } Router::new() .route("/health/liveness", get(liveness_handler)) .route("/health/readiness", get(readiness_handler)) .route("/health/startup", get(startup_handler)) .route( "/resilience/circuit-breaker/status", get(circuit_breaker_status_handler), ) .route( "/resilience/rate-limit/status", get(rate_limit_status_handler), ) .route("/resilience/timeout/config", get(timeout_config_handler)) .route("/resilience/retry/config", get(retry_config_handler)) .route( "/health", get(|| async { axum::Json(serde_json::json!({"status": "healthy"})) }), ) .route("/health/backends", get(backend_services_handler)) .with_state(state) } #[tokio::test] async fn test_gateway_liveness_probe() { let state = MockGatewayHealthState::new(); let app = create_gateway_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/health/liveness") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); } #[tokio::test] async fn test_gateway_readiness_probe_healthy() { let state = MockGatewayHealthState::new(); state.set_healthy(true).await; let app = create_gateway_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/health/readiness") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); } #[tokio::test] async fn test_gateway_readiness_probe_unhealthy() { let state = MockGatewayHealthState::new(); state.set_healthy(false).await; let app = create_gateway_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/health/readiness") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); } #[tokio::test] async fn test_gateway_startup_probe() { let state = MockGatewayHealthState::new(); state.mark_startup_complete().await; let app = create_gateway_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/health/startup") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); } #[tokio::test] async fn test_simple_health_endpoint() { let state = MockGatewayHealthState::new(); let app = create_gateway_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/health") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); // Verify JSON response structure let body = axum::body::to_bytes(response.into_body(), usize::MAX) .await .unwrap(); let json: serde_json::Value = serde_json::from_slice(&body).expect("INVARIANT: Deserialization should succeed for valid JSON"); assert_eq!(json["status"], "healthy"); assert_eq!(json.get("status").expect("INVARIANT: Key should exist in map"), "healthy"); } #[tokio::test] async fn test_gateway_service_startup_transition() { let state = MockGatewayHealthState::new(); *state.startup_complete.write().await = false; // Not ready initially assert!(!state.is_startup_complete().await); // Simulate startup tokio::time::sleep(Duration::from_millis(50)).await; state.mark_startup_complete().await; assert!(state.is_startup_complete().await); } #[tokio::test] async fn test_gateway_circuit_breaker_status() { let state = MockGatewayHealthState::new(); let app = create_gateway_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/resilience/circuit-breaker/status") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); } #[tokio::test] async fn test_gateway_circuit_breaker_open() { let state = MockGatewayHealthState::new(); state.set_circuit_breaker_open(true).await; let app = create_gateway_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/resilience/circuit-breaker/status") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); // Circuit breaker open should still return 200 with state info } #[tokio::test] async fn test_gateway_rate_limit_status() { let state = MockGatewayHealthState::new(); let app = create_gateway_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/resilience/rate-limit/status") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); } #[tokio::test] async fn test_gateway_timeout_config() { let state = MockGatewayHealthState::new(); let app = create_gateway_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/resilience/timeout/config") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); } #[tokio::test] async fn test_gateway_retry_config() { let state = MockGatewayHealthState::new(); let app = create_gateway_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/resilience/retry/config") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); } #[tokio::test] async fn test_gateway_backend_services_all_up() { let state = MockGatewayHealthState::new(); let app = create_gateway_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/health/backends") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); } #[tokio::test] async fn test_gateway_trading_service_down() { let state = MockGatewayHealthState::new(); state.set_trading_service_up(false).await; let app = create_gateway_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/health/backends") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); // Returns 200 with status info even if backend is down } #[tokio::test] async fn test_gateway_all_backends_down() { let state = MockGatewayHealthState::new(); state.set_trading_service_up(false).await; state.set_backtesting_service_up(false).await; state.set_ml_service_up(false).await; let app = create_gateway_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/health/backends") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); // Still returns 200 to report status } #[tokio::test] async fn test_gateway_health_check_latency() { let state = MockGatewayHealthState::new(); let app = create_gateway_health_router(state.clone()); let start = Instant::now(); let response = app .oneshot( Request::builder() .uri("/health/liveness") .body(Body::empty()) .unwrap(), ) .await .unwrap(); let latency = start.elapsed(); assert_eq!(response.status(), StatusCode::OK); assert!( latency < Duration::from_millis(100), "Health check latency: {:?}", latency ); } #[tokio::test] async fn test_gateway_concurrent_health_checks() { let state = MockGatewayHealthState::new(); let mut handles = vec![]; for _ in 0..100 { let state_clone = state.clone(); let handle = tokio::spawn(async move { let app = create_gateway_health_router(state_clone); let response = app .oneshot( Request::builder() .uri("/health/liveness") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); }); handles.push(handle); } for handle in handles { handle.await.unwrap(); } } #[tokio::test] async fn test_gateway_health_during_shutdown() { let state = MockGatewayHealthState::new(); // Graceful shutdown state.set_healthy(false).await; let app = create_gateway_health_router(state.clone()); // Readiness check fails let response = app .clone() .oneshot( Request::builder() .uri("/health/readiness") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); // Liveness check still passes let response = app .oneshot( Request::builder() .uri("/health/liveness") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); } #[tokio::test] async fn test_gateway_rapid_health_checks() { let state = MockGatewayHealthState::new(); let start = Instant::now(); for _ in 0..1000 { let app = create_gateway_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/health/liveness") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); } let duration = start.elapsed(); assert!( duration < Duration::from_secs(1), "1000 health checks took: {:?}", duration ); } #[tokio::test] async fn test_gateway_partial_backend_failure() { let state = MockGatewayHealthState::new(); // Only trading service down state.set_trading_service_up(false).await; state.set_backtesting_service_up(true).await; state.set_ml_service_up(true).await; let app = create_gateway_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/health/backends") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); } #[tokio::test] async fn test_gateway_recovery_after_failure() { let state = MockGatewayHealthState::new(); // Service fails state.set_healthy(false).await; let app = create_gateway_health_router(state.clone()); let response = app .clone() .oneshot( Request::builder() .uri("/health/readiness") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); // Service recovers state.set_healthy(true).await; let response = app .oneshot( Request::builder() .uri("/health/readiness") .body(Body::empty()) .unwrap(), ) .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); } #[tokio::test] async fn test_gateway_rate_limiter_unhealthy() { let state = MockGatewayHealthState::new(); state.set_rate_limiter_healthy(false).await; let app = create_gateway_health_router(state.clone()); let response = app .oneshot( Request::builder() .uri("/resilience/rate-limit/status") .body(Body::empty()) .unwrap(), ) .await .unwrap(); // Still returns 200 with status info assert_eq!(response.status(), StatusCode::OK); }