eae3c31e53
Patterns applied: - Pattern 2: Float comparison (2x: utils.rs, var_edge_cases_tests.rs) - Pattern 7: Date/time construction (2x: production_streaming.rs, streaming.rs) - Pattern 1: Duration/time ops (2x: rate limiter, semaphore) - Pattern 4: Optional field access (1x: position_tracker.rs) Changes: - data/src/utils.rs: Float sort with NaN handling - data/src/providers/benzinga/production_streaming.rs: Rate limiter + semaphore + date/time - data/src/providers/benzinga/streaming.rs: Date/time construction - risk/src/position_tracker.rs: Emergency fallback counter - risk/tests/var_edge_cases_tests.rs: Test helper float sort Test impact: 0 failures (182/182 passing) Compilation: Clean (0 errors, 0 warnings) Time: 25 min (44% under budget)
534 lines
14 KiB
Rust
534 lines
14 KiB
Rust
//! Comprehensive health check tests for Backtesting Service
|
|
//!
|
|
//! Tests cover:
|
|
//! - HTTP /health and /ready endpoints
|
|
//! - Service startup state transitions
|
|
//! - Database connectivity checks
|
|
//! - Storage backend availability
|
|
//! - Data replay capability
|
|
//! - Dependency cascade failures
|
|
//! - Health check latency
|
|
//! - Concurrent health checks
|
|
//! - Health during backtest execution
|
|
//! - Resource exhaustion scenarios
|
|
|
|
use axum::{
|
|
body::Body,
|
|
http::{Request, StatusCode},
|
|
};
|
|
use std::sync::Arc;
|
|
use std::time::{Duration, Instant};
|
|
use tokio::sync::RwLock;
|
|
use tower::ServiceExt;
|
|
|
|
/// Mock health state for backtesting service
|
|
#[derive(Clone)]
|
|
struct MockBacktestHealthState {
|
|
healthy: Arc<RwLock<bool>>,
|
|
ready: Arc<RwLock<bool>>,
|
|
database_connected: Arc<RwLock<bool>>,
|
|
storage_available: Arc<RwLock<bool>>,
|
|
backtest_running: Arc<RwLock<bool>>,
|
|
}
|
|
|
|
impl MockBacktestHealthState {
|
|
fn new() -> Self {
|
|
Self {
|
|
healthy: Arc::new(RwLock::new(true)),
|
|
ready: Arc::new(RwLock::new(true)),
|
|
database_connected: Arc::new(RwLock::new(true)),
|
|
storage_available: Arc::new(RwLock::new(true)),
|
|
backtest_running: Arc::new(RwLock::new(false)),
|
|
}
|
|
}
|
|
|
|
async fn set_healthy(&self, healthy: bool) {
|
|
*self.healthy.write().await = healthy;
|
|
}
|
|
|
|
async fn set_ready(&self, ready: bool) {
|
|
*self.ready.write().await = ready;
|
|
}
|
|
|
|
async fn set_database_connected(&self, connected: bool) {
|
|
*self.database_connected.write().await = connected;
|
|
}
|
|
|
|
async fn set_storage_available(&self, available: bool) {
|
|
*self.storage_available.write().await = available;
|
|
}
|
|
|
|
async fn set_backtest_running(&self, running: bool) {
|
|
*self.backtest_running.write().await = running;
|
|
}
|
|
|
|
async fn is_healthy(&self) -> bool {
|
|
*self.healthy.read().await
|
|
}
|
|
|
|
async fn is_ready(&self) -> bool {
|
|
*self.ready.read().await
|
|
}
|
|
|
|
async fn is_database_connected(&self) -> bool {
|
|
*self.database_connected.read().await
|
|
}
|
|
|
|
async fn is_storage_available(&self) -> bool {
|
|
*self.storage_available.read().await
|
|
}
|
|
|
|
async fn is_backtest_running(&self) -> bool {
|
|
*self.backtest_running.read().await
|
|
}
|
|
}
|
|
|
|
/// Create health router for backtesting service
|
|
fn create_backtest_health_router(state: MockBacktestHealthState) -> axum::Router {
|
|
use axum::{extract::State, routing::get, Json, Router};
|
|
use serde_json::json;
|
|
|
|
async fn health_handler(
|
|
State(state): State<MockBacktestHealthState>,
|
|
) -> Result<Json<serde_json::Value>, StatusCode> {
|
|
if state.is_healthy().await {
|
|
Ok(Json(json!({
|
|
"status": "healthy",
|
|
"service": "backtesting",
|
|
"version": env!("CARGO_PKG_VERSION")
|
|
})))
|
|
} else {
|
|
Err(StatusCode::SERVICE_UNAVAILABLE)
|
|
}
|
|
}
|
|
|
|
async fn ready_handler(
|
|
State(state): State<MockBacktestHealthState>,
|
|
) -> Result<Json<serde_json::Value>, StatusCode> {
|
|
if state.is_ready().await {
|
|
Ok(Json(json!({
|
|
"status": "ready",
|
|
"service": "backtesting",
|
|
"version": env!("CARGO_PKG_VERSION")
|
|
})))
|
|
} else {
|
|
Err(StatusCode::SERVICE_UNAVAILABLE)
|
|
}
|
|
}
|
|
|
|
async fn deep_health_handler(
|
|
State(state): State<MockBacktestHealthState>,
|
|
) -> Result<Json<serde_json::Value>, StatusCode> {
|
|
let db_ok = state.is_database_connected().await;
|
|
let storage_ok = state.is_storage_available().await;
|
|
let healthy = state.is_healthy().await;
|
|
let backtest_running = state.is_backtest_running().await;
|
|
|
|
if healthy && db_ok && storage_ok {
|
|
Ok(Json(json!({
|
|
"status": "healthy",
|
|
"service": "backtesting",
|
|
"version": env!("CARGO_PKG_VERSION"),
|
|
"dependencies": {
|
|
"database": "ok",
|
|
"storage": "ok"
|
|
},
|
|
"backtest_running": backtest_running
|
|
})))
|
|
} else {
|
|
Err(StatusCode::SERVICE_UNAVAILABLE)
|
|
}
|
|
}
|
|
|
|
Router::new()
|
|
.route("/health", get(health_handler))
|
|
.route("/ready", get(ready_handler))
|
|
.route("/health/deep", get(deep_health_handler))
|
|
.with_state(state)
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_backtest_health_basic_healthy() {
|
|
let state = MockBacktestHealthState::new();
|
|
let app = create_backtest_health_router(state.clone());
|
|
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/health")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(response.status(), StatusCode::OK);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_backtest_health_unhealthy() {
|
|
let state = MockBacktestHealthState::new();
|
|
state.set_healthy(false).await;
|
|
let app = create_backtest_health_router(state.clone());
|
|
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/health")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_backtest_readiness_check() {
|
|
let state = MockBacktestHealthState::new();
|
|
let app = create_backtest_health_router(state.clone());
|
|
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/ready")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(response.status(), StatusCode::OK);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_backtest_service_startup() {
|
|
let state = MockBacktestHealthState::new();
|
|
state.set_ready(false).await;
|
|
|
|
// Initially not ready
|
|
assert!(!state.is_ready().await);
|
|
assert!(state.is_healthy().await);
|
|
|
|
// Simulate startup
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
state.set_ready(true).await;
|
|
|
|
assert!(state.is_ready().await);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_backtest_database_disconnection() {
|
|
let state = MockBacktestHealthState::new();
|
|
state.set_database_connected(false).await;
|
|
let app = create_backtest_health_router(state.clone());
|
|
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/health/deep")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_backtest_storage_unavailable() {
|
|
let state = MockBacktestHealthState::new();
|
|
state.set_storage_available(false).await;
|
|
let app = create_backtest_health_router(state.clone());
|
|
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/health/deep")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_backtest_health_during_execution() {
|
|
let state = MockBacktestHealthState::new();
|
|
state.set_backtest_running(true).await;
|
|
|
|
let app = create_backtest_health_router(state.clone());
|
|
|
|
// Service should still be healthy during backtest execution
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/health")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(response.status(), StatusCode::OK);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_backtest_dependency_cascade_failure() {
|
|
let state = MockBacktestHealthState::new();
|
|
|
|
// Both dependencies fail
|
|
state.set_database_connected(false).await;
|
|
state.set_storage_available(false).await;
|
|
|
|
let app = create_backtest_health_router(state.clone());
|
|
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/health/deep")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_backtest_health_check_latency() {
|
|
let state = MockBacktestHealthState::new();
|
|
let app = create_backtest_health_router(state.clone());
|
|
|
|
let start = Instant::now();
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/health")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let latency = start.elapsed();
|
|
|
|
assert_eq!(response.status(), StatusCode::OK);
|
|
assert!(
|
|
latency < Duration::from_millis(100),
|
|
"Health check latency: {:?}",
|
|
latency
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_backtest_concurrent_health_checks() {
|
|
let state = MockBacktestHealthState::new();
|
|
|
|
let mut handles = vec![];
|
|
for _ in 0..100 {
|
|
let state_clone = state.clone();
|
|
let handle = tokio::spawn(async move {
|
|
let app = create_backtest_health_router(state_clone);
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/health")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(response.status(), StatusCode::OK);
|
|
});
|
|
handles.push(handle);
|
|
}
|
|
|
|
for handle in handles {
|
|
handle.await.unwrap();
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_backtest_health_during_shutdown() {
|
|
let state = MockBacktestHealthState::new();
|
|
|
|
// Graceful shutdown
|
|
state.set_ready(false).await;
|
|
state.set_healthy(true).await;
|
|
|
|
let app = create_backtest_health_router(state.clone());
|
|
|
|
// Ready check fails
|
|
let response = app
|
|
.clone()
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/ready")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
|
|
|
// Health check passes
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/health")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(response.status(), StatusCode::OK);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_backtest_rapid_health_checks() {
|
|
let state = MockBacktestHealthState::new();
|
|
|
|
let start = Instant::now();
|
|
for _ in 0..500 {
|
|
let app = create_backtest_health_router(state.clone());
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/health")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(response.status(), StatusCode::OK);
|
|
}
|
|
let duration = start.elapsed();
|
|
|
|
assert!(
|
|
duration < Duration::from_secs(1),
|
|
"500 health checks took: {:?}",
|
|
duration
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_backtest_deep_vs_shallow_health() {
|
|
let state = MockBacktestHealthState::new();
|
|
let app = create_backtest_health_router(state.clone());
|
|
|
|
// Shallow health
|
|
let start = Instant::now();
|
|
let response = app
|
|
.clone()
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/health")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let shallow_latency = start.elapsed();
|
|
assert_eq!(response.status(), StatusCode::OK);
|
|
|
|
// Deep health
|
|
let start = Instant::now();
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/health/deep")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let deep_latency = start.elapsed();
|
|
assert_eq!(response.status(), StatusCode::OK);
|
|
|
|
assert!(shallow_latency < Duration::from_millis(50));
|
|
assert!(deep_latency < Duration::from_millis(100));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_backtest_partial_availability() {
|
|
let state = MockBacktestHealthState::new();
|
|
|
|
// Database down, storage up
|
|
state.set_database_connected(false).await;
|
|
state.set_storage_available(true).await;
|
|
let app = create_backtest_health_router(state.clone());
|
|
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/health/deep")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_backtest_recovery_after_failure() {
|
|
let state = MockBacktestHealthState::new();
|
|
|
|
// Service fails
|
|
state.set_healthy(false).await;
|
|
let app = create_backtest_health_router(state.clone());
|
|
|
|
let response = app
|
|
.clone()
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/health")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
|
|
|
// Service recovers
|
|
state.set_healthy(true).await;
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/health")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(response.status(), StatusCode::OK);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_backtest_health_json_format() {
|
|
let state = MockBacktestHealthState::new();
|
|
let app = create_backtest_health_router(state.clone());
|
|
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/health")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(response.status(), StatusCode::OK);
|
|
|
|
let content_type = response.headers().get("content-type");
|
|
assert!(content_type.is_some());
|
|
let content_type_str = content_type.unwrap().to_str().expect("INVARIANT: Path should be valid UTF-8");
|
|
assert!(content_type_str.contains("application/json"));
|
|
}
|