foxhunt/services/api/tests/health_check_tests.rs

//! Comprehensive health check tests for API Gateway
//!
//! Tests cover:
//! - HTTP /health/liveness, /health/readiness, /health/startup endpoints
//! - Backend service health checks
//! - Rate limiter status
//! - Circuit breaker status
//! - Service startup transitions
//! - Backend service cascade failures
//! - Health check latency
//! - Concurrent health checks
//! - Health during high load
//! - Resilience endpoint tests

use axum::{
    body::Body,
    http::{Request, StatusCode},
};
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::RwLock;
use tower::ServiceExt;

/// Mock health state for API Gateway
#[derive(Clone)]
struct MockGatewayHealthState {
    startup_complete: Arc<RwLock<bool>>,
    service_healthy: Arc<RwLock<bool>>,
    trading_service_up: Arc<RwLock<bool>>,
    backtesting_service_up: Arc<RwLock<bool>>,
    ml_service_up: Arc<RwLock<bool>>,
    rate_limiter_healthy: Arc<RwLock<bool>>,
    circuit_breaker_open: Arc<RwLock<bool>>,
}

impl MockGatewayHealthState {
    fn new() -> Self {
        Self {
            startup_complete: Arc::new(RwLock::new(true)),
            service_healthy: Arc::new(RwLock::new(true)),
            trading_service_up: Arc::new(RwLock::new(true)),
            backtesting_service_up: Arc::new(RwLock::new(true)),
            ml_service_up: Arc::new(RwLock::new(true)),
            rate_limiter_healthy: Arc::new(RwLock::new(true)),
            circuit_breaker_open: Arc::new(RwLock::new(false)),
        }
    }

    async fn mark_startup_complete(&self) {
        *self.startup_complete.write().await = true;
    }

    async fn set_healthy(&self, healthy: bool) {
        *self.service_healthy.write().await = healthy;
    }

    async fn set_trading_service_up(&self, up: bool) {
        *self.trading_service_up.write().await = up;
    }

    async fn set_backtesting_service_up(&self, up: bool) {
        *self.backtesting_service_up.write().await = up;
    }

    async fn set_ml_service_up(&self, up: bool) {
        *self.ml_service_up.write().await = up;
    }

    async fn set_rate_limiter_healthy(&self, healthy: bool) {
        *self.rate_limiter_healthy.write().await = healthy;
    }

    async fn set_circuit_breaker_open(&self, open: bool) {
        *self.circuit_breaker_open.write().await = open;
    }

    async fn is_startup_complete(&self) -> bool {
        *self.startup_complete.read().await
    }

    async fn is_healthy(&self) -> bool {
        *self.service_healthy.read().await
    }

    async fn is_trading_service_up(&self) -> bool {
        *self.trading_service_up.read().await
    }

    async fn is_backtesting_service_up(&self) -> bool {
        *self.backtesting_service_up.read().await
    }

    async fn is_ml_service_up(&self) -> bool {
        *self.ml_service_up.read().await
    }

    async fn is_rate_limiter_healthy(&self) -> bool {
        *self.rate_limiter_healthy.read().await
    }

    async fn is_circuit_breaker_open(&self) -> bool {
        *self.circuit_breaker_open.read().await
    }
}

/// Create health router for API Gateway
fn create_gateway_health_router(state: MockGatewayHealthState) -> axum::Router {
    use axum::{extract::State, routing::get, Json, Router};
    use serde_json::json;

    async fn liveness_handler() -> &'static str {
        "OK"
    }

    async fn readiness_handler(
        State(state): State<MockGatewayHealthState>,
    ) -> Result<String, StatusCode> {
        if state.is_healthy().await {
            Ok("READY".to_string())
        } else {
            Err(StatusCode::SERVICE_UNAVAILABLE)
        }
    }

    async fn startup_handler(
        State(state): State<MockGatewayHealthState>,
    ) -> Result<String, StatusCode> {
        if state.is_startup_complete().await {
            Ok("READY".to_string())
        } else {
            Err(StatusCode::SERVICE_UNAVAILABLE)
        }
    }

    async fn circuit_breaker_status_handler(
        State(state): State<MockGatewayHealthState>,
    ) -> Json<serde_json::Value> {
        let is_open = state.is_circuit_breaker_open().await;
        Json(json!({
            "state": if is_open { "open" } else { "closed" },
            "failure_count": if is_open { 5 } else { 0 },
            "success_count": 100,
            "threshold": 5,
            "timeout_seconds": 60
        }))
    }

    async fn rate_limit_status_handler(
        State(state): State<MockGatewayHealthState>,
    ) -> Json<serde_json::Value> {
        let healthy = state.is_rate_limiter_healthy().await;
        Json(json!({
            "enabled": true,
            "requests_per_minute": 1000,
            "current_usage": if healthy { 0 } else { 1000 },
            "burst_size": 100
        }))
    }

    async fn timeout_config_handler() -> Json<serde_json::Value> {
        Json(json!({
            "request_timeout_ms": 5000,
            "connection_timeout_ms": 3000,
            "idle_timeout_ms": 60000
        }))
    }

    async fn retry_config_handler() -> Json<serde_json::Value> {
        Json(json!({
            "max_retries": 3,
            "retry_delay_ms": 100,
            "backoff_multiplier": 2.0,
            "max_backoff_ms": 10000
        }))
    }

    async fn backend_services_handler(
        State(state): State<MockGatewayHealthState>,
    ) -> Json<serde_json::Value> {
        let trading_up = state.is_trading_service_up().await;
        let backtesting_up = state.is_backtesting_service_up().await;
        let ml_up = state.is_ml_service_up().await;

        Json(json!({
            "trading_service": if trading_up { "up" } else { "down" },
            "backtesting_service": if backtesting_up { "up" } else { "down" },
            "ml_training_service": if ml_up { "up" } else { "down" }
        }))
    }

    Router::new()
        .route("/health/liveness", get(liveness_handler))
        .route("/health/readiness", get(readiness_handler))
        .route("/health/startup", get(startup_handler))
        .route(
            "/resilience/circuit-breaker/status",
            get(circuit_breaker_status_handler),
        )
        .route(
            "/resilience/rate-limit/status",
            get(rate_limit_status_handler),
        )
        .route("/resilience/timeout/config", get(timeout_config_handler))
        .route("/resilience/retry/config", get(retry_config_handler))
        .route(
            "/health",
            get(|| async { axum::Json(serde_json::json!({"status": "healthy"})) }),
        )
        .route("/health/backends", get(backend_services_handler))
        .with_state(state)
}

#[tokio::test]
async fn test_gateway_liveness_probe() {
    let state = MockGatewayHealthState::new();
    let app = create_gateway_health_router(state.clone());

    let response = app
        .oneshot(
            Request::builder()
                .uri("/health/liveness")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_eq!(response.status(), StatusCode::OK);
}

#[tokio::test]
async fn test_gateway_readiness_probe_healthy() {
    let state = MockGatewayHealthState::new();
    state.set_healthy(true).await;
    let app = create_gateway_health_router(state.clone());

    let response = app
        .oneshot(
            Request::builder()
                .uri("/health/readiness")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_eq!(response.status(), StatusCode::OK);
}

#[tokio::test]
async fn test_gateway_readiness_probe_unhealthy() {
    let state = MockGatewayHealthState::new();
    state.set_healthy(false).await;
    let app = create_gateway_health_router(state.clone());

    let response = app
        .oneshot(
            Request::builder()
                .uri("/health/readiness")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
}

#[tokio::test]
async fn test_gateway_startup_probe() {
    let state = MockGatewayHealthState::new();
    state.mark_startup_complete().await;
    let app = create_gateway_health_router(state.clone());

    let response = app
        .oneshot(
            Request::builder()
                .uri("/health/startup")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_eq!(response.status(), StatusCode::OK);
}

#[tokio::test]
async fn test_simple_health_endpoint() {
    let state = MockGatewayHealthState::new();
    let app = create_gateway_health_router(state.clone());

    let response = app
        .oneshot(
            Request::builder()
                .uri("/health")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_eq!(response.status(), StatusCode::OK);

    // Verify JSON response structure
    let body = axum::body::to_bytes(response.into_body(), usize::MAX)
        .await
        .unwrap();
    let json: serde_json::Value = serde_json::from_slice(&body).expect("INVARIANT: Deserialization should succeed for valid JSON");
    assert_eq!(json["status"], "healthy");
    assert_eq!(json.get("status").expect("INVARIANT: Key should exist in map"), "healthy");
}

#[tokio::test]
async fn test_gateway_service_startup_transition() {
    let state = MockGatewayHealthState::new();
    *state.startup_complete.write().await = false;

    // Not ready initially
    assert!(!state.is_startup_complete().await);

    // Simulate startup
    tokio::time::sleep(Duration::from_millis(50)).await;
    state.mark_startup_complete().await;

    assert!(state.is_startup_complete().await);
}

#[tokio::test]
async fn test_gateway_circuit_breaker_status() {
    let state = MockGatewayHealthState::new();
    let app = create_gateway_health_router(state.clone());

    let response = app
        .oneshot(
            Request::builder()
                .uri("/resilience/circuit-breaker/status")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_eq!(response.status(), StatusCode::OK);
}

#[tokio::test]
async fn test_gateway_circuit_breaker_open() {
    let state = MockGatewayHealthState::new();
    state.set_circuit_breaker_open(true).await;
    let app = create_gateway_health_router(state.clone());

    let response = app
        .oneshot(
            Request::builder()
                .uri("/resilience/circuit-breaker/status")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_eq!(response.status(), StatusCode::OK);
    // Circuit breaker open should still return 200 with state info
}

#[tokio::test]
async fn test_gateway_rate_limit_status() {
    let state = MockGatewayHealthState::new();
    let app = create_gateway_health_router(state.clone());

    let response = app
        .oneshot(
            Request::builder()
                .uri("/resilience/rate-limit/status")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_eq!(response.status(), StatusCode::OK);
}

#[tokio::test]
async fn test_gateway_timeout_config() {
    let state = MockGatewayHealthState::new();
    let app = create_gateway_health_router(state.clone());

    let response = app
        .oneshot(
            Request::builder()
                .uri("/resilience/timeout/config")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_eq!(response.status(), StatusCode::OK);
}

#[tokio::test]
async fn test_gateway_retry_config() {
    let state = MockGatewayHealthState::new();
    let app = create_gateway_health_router(state.clone());

    let response = app
        .oneshot(
            Request::builder()
                .uri("/resilience/retry/config")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_eq!(response.status(), StatusCode::OK);
}

#[tokio::test]
async fn test_gateway_backend_services_all_up() {
    let state = MockGatewayHealthState::new();
    let app = create_gateway_health_router(state.clone());

    let response = app
        .oneshot(
            Request::builder()
                .uri("/health/backends")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_eq!(response.status(), StatusCode::OK);
}

#[tokio::test]
async fn test_gateway_trading_service_down() {
    let state = MockGatewayHealthState::new();
    state.set_trading_service_up(false).await;
    let app = create_gateway_health_router(state.clone());

    let response = app
        .oneshot(
            Request::builder()
                .uri("/health/backends")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_eq!(response.status(), StatusCode::OK);
    // Returns 200 with status info even if backend is down
}

#[tokio::test]
async fn test_gateway_all_backends_down() {
    let state = MockGatewayHealthState::new();
    state.set_trading_service_up(false).await;
    state.set_backtesting_service_up(false).await;
    state.set_ml_service_up(false).await;

    let app = create_gateway_health_router(state.clone());

    let response = app
        .oneshot(
            Request::builder()
                .uri("/health/backends")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_eq!(response.status(), StatusCode::OK);
    // Still returns 200 to report status
}

#[tokio::test]
async fn test_gateway_health_check_latency() {
    let state = MockGatewayHealthState::new();
    let app = create_gateway_health_router(state.clone());

    let start = Instant::now();
    let response = app
        .oneshot(
            Request::builder()
                .uri("/health/liveness")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let latency = start.elapsed();

    assert_eq!(response.status(), StatusCode::OK);
    assert!(
        latency < Duration::from_millis(100),
        "Health check latency: {:?}",
        latency
    );
}

#[tokio::test]
async fn test_gateway_concurrent_health_checks() {
    let state = MockGatewayHealthState::new();

    let mut handles = vec![];
    for _ in 0..100 {
        let state_clone = state.clone();
        let handle = tokio::spawn(async move {
            let app = create_gateway_health_router(state_clone);
            let response = app
                .oneshot(
                    Request::builder()
                        .uri("/health/liveness")
                        .body(Body::empty())
                        .unwrap(),
                )
                .await
                .unwrap();
            assert_eq!(response.status(), StatusCode::OK);
        });
        handles.push(handle);
    }

    for handle in handles {
        handle.await.unwrap();
    }
}

#[tokio::test]
async fn test_gateway_health_during_shutdown() {
    let state = MockGatewayHealthState::new();

    // Graceful shutdown
    state.set_healthy(false).await;
    let app = create_gateway_health_router(state.clone());

    // Readiness check fails
    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .uri("/health/readiness")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);

    // Liveness check still passes
    let response = app
        .oneshot(
            Request::builder()
                .uri("/health/liveness")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    assert_eq!(response.status(), StatusCode::OK);
}

#[tokio::test]
async fn test_gateway_rapid_health_checks() {
    let state = MockGatewayHealthState::new();

    let start = Instant::now();
    for _ in 0..1000 {
        let app = create_gateway_health_router(state.clone());
        let response = app
            .oneshot(
                Request::builder()
                    .uri("/health/liveness")
                    .body(Body::empty())
                    .unwrap(),
            )
            .await
            .unwrap();
        assert_eq!(response.status(), StatusCode::OK);
    }
    let duration = start.elapsed();

    assert!(
        duration < Duration::from_secs(1),
        "1000 health checks took: {:?}",
        duration
    );
}

#[tokio::test]
async fn test_gateway_partial_backend_failure() {
    let state = MockGatewayHealthState::new();

    // Only trading service down
    state.set_trading_service_up(false).await;
    state.set_backtesting_service_up(true).await;
    state.set_ml_service_up(true).await;

    let app = create_gateway_health_router(state.clone());

    let response = app
        .oneshot(
            Request::builder()
                .uri("/health/backends")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    assert_eq!(response.status(), StatusCode::OK);
}

#[tokio::test]
async fn test_gateway_recovery_after_failure() {
    let state = MockGatewayHealthState::new();

    // Service fails
    state.set_healthy(false).await;
    let app = create_gateway_health_router(state.clone());

    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .uri("/health/readiness")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);

    // Service recovers
    state.set_healthy(true).await;
    let response = app
        .oneshot(
            Request::builder()
                .uri("/health/readiness")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    assert_eq!(response.status(), StatusCode::OK);
}

#[tokio::test]
async fn test_gateway_rate_limiter_unhealthy() {
    let state = MockGatewayHealthState::new();
    state.set_rate_limiter_healthy(false).await;
    let app = create_gateway_health_router(state.clone());

    let response = app
        .oneshot(
            Request::builder()
                .uri("/resilience/rate-limit/status")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();

    // Still returns 200 with status info
    assert_eq!(response.status(), StatusCode::OK);
}