foxhunt/services/AGENT_22_TEST_PATTERNS.md

Agent 22: Health Check Test Patterns Reference

Test Pattern Examples

1. Basic Health Check Pattern

#[tokio::test]
async fn test_health_check_basic_healthy() {
    let state = MockHealthState::new();
    let app = create_health_router(state.clone());

    let response = app
        .oneshot(Request::builder().uri("/health").body(Body::empty()).unwrap())
        .await
        .unwrap();

    assert_eq!(response.status(), StatusCode::OK);
}

2. Dependency Failure Pattern

#[tokio::test]
async fn test_database_disconnection() {
    let state = MockHealthState::new();
    state.set_database_connected(false).await;
    let app = create_health_router(state.clone());

    let response = app
        .oneshot(Request::builder().uri("/health/deep").body(Body::empty()).unwrap())
        .await
        .unwrap();

    assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
}

3. Latency Measurement Pattern

#[tokio::test]
async fn test_health_check_latency() {
    let state = MockHealthState::new();
    let app = create_health_router(state.clone());

    let start = Instant::now();
    let response = app
        .oneshot(Request::builder().uri("/health").body(Body::empty()).unwrap())
        .await
        .unwrap();
    let latency = start.elapsed();

    assert_eq!(response.status(), StatusCode::OK);
    assert!(latency < Duration::from_millis(100), "Health check latency: {:?}", latency);
}

4. Concurrent Testing Pattern

#[tokio::test]
async fn test_concurrent_health_checks() {
    let state = MockHealthState::new();

    let mut handles = vec![];
    for _ in 0..100 {
        let state_clone = state.clone();
        let handle = tokio::spawn(async move {
            let app = create_health_router(state_clone);
            let response = app
                .oneshot(Request::builder().uri("/health").body(Body::empty()).unwrap())
                .await
                .unwrap();
            assert_eq!(response.status(), StatusCode::OK);
        });
        handles.push(handle);
    }

    for handle in handles {
        handle.await.unwrap();
    }
}

5. State Transition Pattern

#[tokio::test]
async fn test_service_startup_transition() {
    let state = MockHealthState::new();
    state.set_ready(false).await;

    // Initially not ready
    assert!(!state.is_ready().await);
    assert!(state.is_healthy().await);

    // Simulate initialization
    tokio::time::sleep(Duration::from_millis(50)).await;
    state.set_ready(true).await;

    assert!(state.is_ready().await);
}

6. Recovery Pattern

#[tokio::test]
async fn test_health_recovery_after_failure() {
    let state = MockHealthState::new();

    // Service fails
    state.set_healthy(false).await;
    let app = create_health_router(state.clone());

    let response = app.clone()
        .oneshot(Request::builder().uri("/health").body(Body::empty()).unwrap())
        .await
        .unwrap();
    assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);

    // Service recovers
    state.set_healthy(true).await;
    let response = app
        .oneshot(Request::builder().uri("/health").body(Body::empty()).unwrap())
        .await
        .unwrap();
    assert_eq!(response.status(), StatusCode::OK);
}

Mock Health State Implementation

Trading Service Mock

#[derive(Clone)]
struct MockHealthState {
    healthy: Arc<RwLock<bool>>,
    ready: Arc<RwLock<bool>>,
    database_connected: Arc<RwLock<bool>>,
    redis_connected: Arc<RwLock<bool>>,
}

ML Training Service Mock

#[derive(Clone)]
struct MockMlHealthState {
    healthy: Arc<RwLock<bool>>,
    ready: Arc<RwLock<bool>>,
    gpu_available: Arc<RwLock<bool>>,
    checkpoints_accessible: Arc<RwLock<bool>>,
    database_connected: Arc<RwLock<bool>>,
    training_active: Arc<RwLock<bool>>,
    gpu_memory_available: Arc<RwLock<bool>>,
}

API Gateway Mock

#[derive(Clone)]
struct MockGatewayHealthState {
    startup_complete: Arc<RwLock<bool>>,
    service_healthy: Arc<RwLock<bool>>,
    trading_service_up: Arc<RwLock<bool>>,
    backtesting_service_up: Arc<RwLock<bool>>,
    ml_service_up: Arc<RwLock<bool>>,
    rate_limiter_healthy: Arc<RwLock<bool>>,
    circuit_breaker_open: Arc<RwLock<bool>>,
}

Edge Cases Covered

Service-Specific Edge Cases

Trading Service:

• Database down + Redis up (partial degradation)
• Cascade failure (database + Redis)
• 1000 rapid health checks
• 100 concurrent health checks
• Graceful shutdown (liveness OK, readiness FAIL)

Backtesting Service:

• Storage unavailable during backtest
• Database down + storage up
• Health during active backtest
• 500 rapid health checks

ML Training Service:

• GPU available + memory exhausted
• Checkpoints inaccessible + GPU working
• Health during training
• GPU recovery after failure
• 4-way dependency failure

API Gateway:

• Single backend down
• All backends down
• Circuit breaker open
• Rate limiter at capacity
• Kubernetes probe compatibility

Performance Requirements

Test Type	Target	Validation
Health Check Latency	<100ms	✅ 4 tests
Concurrent Requests	100 parallel	✅ 4 tests
Rapid Fire	500-1000 req/s	✅ 4 tests
Deep Health	<100ms	✅ 4 tests

Test Execution

# Run all health check tests
cargo test --test health_check_tests

# Run specific service tests
cargo test -p trading_service --test health_check_tests
cargo test -p backtesting_service --test health_check_tests
cargo test -p ml_training_service --test health_check_tests
cargo test -p api_gateway --test health_check_tests

# Run single test
cargo test test_health_check_latency

# Run with output
cargo test --test health_check_tests -- --nocapture

# Run with single thread (for debugging)
cargo test --test health_check_tests -- --test-threads=1