Copied from api_gateway, removed REST handlers (port 8080), added tonic-web + CORS for grpc-web browser access. Binary renamed: api-gateway → api Changes: - Package name: api-gateway → api - Deleted src/handlers/ (REST ML endpoints on port 8080) - Added tonic-web 0.13 + tower-http CORS layer - Server::builder().accept_http1(true) for grpc-web - CORS_ORIGINS env var (default http://localhost:5173) - Metrics server on port 9091 (axum) preserved - All 95 lib tests pass, 0 clippy warnings - Added services/api to workspace members Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
286 lines
7.9 KiB
Rust
286 lines
7.9 KiB
Rust
//! Health and Resilience HTTP Endpoints
|
|
//!
|
|
//! Provides Kubernetes-compatible health probes and resilience status endpoints:
|
|
//! - /health/liveness - Liveness probe (always OK if service is running)
|
|
//! - /health/readiness - Readiness probe (checks backend services)
|
|
//! - /health/startup - Startup probe (initialization complete)
|
|
//! - /resilience/circuit-breaker/status - Circuit breaker state
|
|
//! - /resilience/rate-limit/status - Rate limiter status
|
|
//! - /resilience/timeout/config - Timeout configuration
|
|
//! - /resilience/retry/config - Retry policy
|
|
|
|
use axum::{extract::State, http::StatusCode, routing::get, Json, Router};
|
|
use serde_json::{json, Value};
|
|
use std::sync::Arc;
|
|
|
|
/// Application state for health endpoints
|
|
#[derive(Clone)]
|
|
pub struct HealthState {
|
|
pub startup_complete: Arc<std::sync::atomic::AtomicBool>,
|
|
pub service_healthy: Arc<std::sync::atomic::AtomicBool>,
|
|
}
|
|
|
|
impl HealthState {
|
|
pub fn new() -> Self {
|
|
Self {
|
|
startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(true)),
|
|
service_healthy: Arc::new(std::sync::atomic::AtomicBool::new(true)),
|
|
}
|
|
}
|
|
|
|
pub fn mark_startup_complete(&self) {
|
|
self.startup_complete
|
|
.store(true, std::sync::atomic::Ordering::SeqCst);
|
|
}
|
|
|
|
pub fn set_healthy(&self, healthy: bool) {
|
|
self.service_healthy
|
|
.store(healthy, std::sync::atomic::Ordering::SeqCst);
|
|
}
|
|
|
|
pub fn is_startup_complete(&self) -> bool {
|
|
self.startup_complete
|
|
.load(std::sync::atomic::Ordering::SeqCst)
|
|
}
|
|
|
|
pub fn is_healthy(&self) -> bool {
|
|
self.service_healthy
|
|
.load(std::sync::atomic::Ordering::SeqCst)
|
|
}
|
|
}
|
|
|
|
impl Default for HealthState {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
/// Liveness probe - always returns OK if service is running
|
|
async fn liveness() -> &'static str {
|
|
"OK"
|
|
}
|
|
|
|
/// Simple health check endpoint - returns JSON status
|
|
async fn health() -> Json<Value> {
|
|
Json(json!({"status": "healthy"}))
|
|
}
|
|
|
|
/// Readiness probe - checks if service is ready to accept traffic
|
|
async fn readiness(State(state): State<HealthState>) -> Result<String, StatusCode> {
|
|
if state.is_healthy() {
|
|
Ok("READY".to_string())
|
|
} else {
|
|
Err(StatusCode::SERVICE_UNAVAILABLE)
|
|
}
|
|
}
|
|
|
|
/// Startup probe - checks if initialization is complete
|
|
async fn startup(State(state): State<HealthState>) -> Result<String, StatusCode> {
|
|
if state.is_startup_complete() {
|
|
Ok("READY".to_string())
|
|
} else {
|
|
Err(StatusCode::SERVICE_UNAVAILABLE)
|
|
}
|
|
}
|
|
|
|
/// Circuit breaker status
|
|
async fn circuit_breaker_status() -> Json<Value> {
|
|
Json(json!({
|
|
"state": "closed",
|
|
"failure_count": 0,
|
|
"success_count": 100,
|
|
"threshold": 5,
|
|
"timeout_seconds": 60
|
|
}))
|
|
}
|
|
|
|
/// Rate limiter status
|
|
async fn rate_limit_status() -> Json<Value> {
|
|
Json(json!({
|
|
"enabled": true,
|
|
"requests_per_minute": 1000,
|
|
"current_usage": 0,
|
|
"burst_size": 100
|
|
}))
|
|
}
|
|
|
|
/// Timeout configuration
|
|
async fn timeout_config() -> Json<Value> {
|
|
Json(json!({
|
|
"request_timeout_ms": 5000,
|
|
"connection_timeout_ms": 3000,
|
|
"idle_timeout_ms": 60000
|
|
}))
|
|
}
|
|
|
|
/// Retry policy configuration
|
|
async fn retry_config() -> Json<Value> {
|
|
Json(json!({
|
|
"max_retries": 3,
|
|
"retry_delay_ms": 100,
|
|
"backoff_multiplier": 2.0,
|
|
"max_backoff_ms": 10000
|
|
}))
|
|
}
|
|
|
|
/// Create health and resilience router
|
|
pub fn health_router(state: HealthState) -> Router {
|
|
Router::new()
|
|
// Simple health endpoint
|
|
.route("/health", get(health))
|
|
// Health probes (Kubernetes-compatible)
|
|
.route("/health/liveness", get(liveness))
|
|
.route("/health/readiness", get(readiness))
|
|
.route("/health/startup", get(startup))
|
|
// Resilience status endpoints
|
|
.route("/resilience/circuit-breaker/status", get(circuit_breaker_status))
|
|
.route("/resilience/rate-limit/status", get(rate_limit_status))
|
|
.route("/resilience/timeout/config", get(timeout_config))
|
|
.route("/resilience/retry/config", get(retry_config))
|
|
.with_state(state)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use axum::body::Body;
|
|
use axum::http::{Request, StatusCode};
|
|
use tower::ServiceExt;
|
|
|
|
#[tokio::test]
|
|
async fn test_liveness_probe() {
|
|
let state = HealthState::new();
|
|
let app = health_router(state);
|
|
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/health/liveness")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(response.status(), StatusCode::OK);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_readiness_probe_healthy() {
|
|
let state = HealthState::new();
|
|
state.set_healthy(true);
|
|
let app = health_router(state);
|
|
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/health/readiness")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(response.status(), StatusCode::OK);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_readiness_probe_unhealthy() {
|
|
let state = HealthState::new();
|
|
state.set_healthy(false);
|
|
let app = health_router(state);
|
|
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/health/readiness")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_startup_probe() {
|
|
let state = HealthState::new();
|
|
state.mark_startup_complete();
|
|
let app = health_router(state);
|
|
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/health/startup")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(response.status(), StatusCode::OK);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_circuit_breaker_status() {
|
|
let state = HealthState::new();
|
|
let app = health_router(state);
|
|
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/resilience/circuit-breaker/status")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(response.status(), StatusCode::OK);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_rate_limit_status() {
|
|
let state = HealthState::new();
|
|
let app = health_router(state);
|
|
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/resilience/rate-limit/status")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(response.status(), StatusCode::OK);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_health_endpoint() {
|
|
let state = HealthState::new();
|
|
let app = health_router(state);
|
|
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/health")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(response.status(), StatusCode::OK);
|
|
|
|
// Verify JSON response
|
|
let body = axum::body::to_bytes(response.into_body(), usize::MAX)
|
|
.await
|
|
.unwrap();
|
|
let json: serde_json::Value = serde_json::from_slice(&body).expect("INVARIANT: Deserialization should succeed for valid JSON");
|
|
assert_eq!(json["status"], "healthy");
|
|
}
|
|
}
|