diff --git a/Cargo.lock b/Cargo.lock index ae3dfef9a..10bc34d0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1516,6 +1516,7 @@ dependencies = [ "anyhow", "async-stream", "async-trait", + "axum 0.7.9", "base64 0.22.1", "chrono", "common", diff --git a/docker-compose.yml b/docker-compose.yml index d82e27618..059c083e0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -177,6 +177,7 @@ services: ports: - "50053:50052" # Map external 50053 to internal 50052 - "9093:9093" # Metrics + - "8083:8080" # Health check endpoint environment: - DATABASE_URL=postgresql://foxhunt:foxhunt_dev_password@postgres:5432/foxhunt - REDIS_URL=redis://redis:6379 @@ -185,6 +186,8 @@ services: - BENZINGA_API_KEY=${BENZINGA_API_KEY:-demo_key_please_replace} - RUST_LOG=info - RUST_BACKTRACE=1 + volumes: + - /tmp/foxhunt/certs:/tmp/foxhunt/certs:ro depends_on: postgres: condition: service_healthy @@ -193,7 +196,7 @@ services: vault: condition: service_healthy healthcheck: - test: ["CMD", "/usr/local/bin/grpc_health_probe", "-addr=localhost:50052"] + test: ["CMD", "curl", "-f", "http://localhost:8080/health"] interval: 10s timeout: 5s start_period: 30s @@ -218,6 +221,10 @@ services: - VAULT_TOKEN=foxhunt-dev-root - RUST_LOG=info - RUST_BACKTRACE=1 + volumes: + - /tmp/foxhunt/certs:/tmp/foxhunt/certs:ro + - /tmp/foxhunt/models:/tmp/foxhunt/models + - /tmp/foxhunt/checkpoints:/tmp/foxhunt/checkpoints depends_on: postgres: condition: service_healthy @@ -260,6 +267,8 @@ services: - ENABLE_AUDIT_LOGGING=true - RUST_LOG=info - RUST_BACKTRACE=1 + volumes: + - /tmp/foxhunt/certs:/tmp/foxhunt/certs:ro depends_on: postgres: condition: service_healthy @@ -270,9 +279,7 @@ services: trading_service: condition: service_healthy backtesting_service: - condition: service_healthy - ml_training_service: - condition: service_healthy + condition: service_started healthcheck: test: ["CMD", "/usr/local/bin/grpc_health_probe", "-addr=localhost:50050"] interval: 10s diff --git a/services/api_gateway/src/main.rs b/services/api_gateway/src/main.rs index 674f7a67f..1076a2e2c 100644 --- a/services/api_gateway/src/main.rs +++ b/services/api_gateway/src/main.rs @@ -119,13 +119,20 @@ async fn main() -> Result<()> { .expect("Failed to create trading service proxy"); info!("✓ Trading service proxy initialized ({})", trading_backend_url); - // Initialize backtesting service proxy - let backtesting_proxy = api_gateway::grpc::BacktestingServiceProxy::new(&backtesting_backend_url) - .await - .expect("Failed to create backtesting service proxy"); - info!("✓ Backtesting service proxy initialized ({})", backtesting_backend_url); + // Initialize backtesting service proxy (optional - graceful degradation) + let backtesting_proxy = match api_gateway::grpc::BacktestingServiceProxy::new(&backtesting_backend_url).await { + Ok(proxy) => { + info!("✓ Backtesting service proxy initialized ({})", backtesting_backend_url); + Some(proxy) + } + Err(e) => { + warn!("⚠ Backtesting service unavailable: {}. API Gateway will run without backtesting endpoints.", e); + warn!(" Backtesting endpoints will return 503 Service Unavailable"); + None + } + }; - // Initialize ML training service proxy + // Initialize ML training service proxy (optional - graceful degradation) let ml_config = api_gateway::grpc::MlTrainingBackendConfig { address: ml_training_backend_url.clone(), connect_timeout_ms: 5000, @@ -133,10 +140,17 @@ async fn main() -> Result<()> { circuit_breaker_failures: 5, circuit_breaker_reset_secs: 30, }; - let ml_training_proxy = api_gateway::grpc::setup_ml_training_proxy(ml_config) - .await - .expect("Failed to create ML training service proxy"); - info!("✓ ML training service proxy initialized ({})", ml_training_backend_url); + let ml_training_proxy = match api_gateway::grpc::setup_ml_training_proxy(ml_config).await { + Ok(proxy) => { + info!("✓ ML training service proxy initialized ({})", ml_training_backend_url); + Some(proxy) + } + Err(e) => { + warn!("⚠ ML Training service unavailable: {}. API Gateway will run without ML endpoints.", e); + warn!(" ML training endpoints will return 503 Service Unavailable"); + None + } + }; // Initialize configuration manager (requires database) let database_url = std::env::var("DATABASE_URL") @@ -189,39 +203,68 @@ async fn main() -> Result<()> { // Setup health check service let (health_reporter, health_service) = tonic_health::server::health_reporter(); + + // Always register trading service (required) health_reporter .set_serving::>() .await; - health_reporter - .set_serving::>() - .await; - health_reporter - .set_serving::>() - .await; - // Build and start server with HTTP/2 optimizations - let server = tonic::transport::Server::builder() + // Conditionally register optional services + if backtesting_proxy.is_some() { + health_reporter + .set_serving::>() + .await; + } + if ml_training_proxy.is_some() { + health_reporter + .set_serving::>() + .await; + } + + // Build server with HTTP/2 optimizations + let mut server_builder = tonic::transport::Server::builder() .max_concurrent_streams(Some(10_000)) .http2_keepalive_interval(Some(Duration::from_secs(30))) .http2_keepalive_timeout(Some(Duration::from_secs(10))) .layer(tower::ServiceBuilder::new() - .layer(tower::layer::util::Identity::new())) // Placeholder for auth interceptor layer - .add_service(health_service) - .add_service(TradingServiceServer::new(trading_proxy)) - .add_service(BacktestingServiceServer::new(backtesting_proxy)) - .add_service(MlTrainingServiceServer::new(ml_training_proxy)) - .serve_with_shutdown(addr, async { - shutdown_rx.await.ok(); - }); + .layer(tower::layer::util::Identity::new())); // Placeholder for auth interceptor layer + // Add health service + let mut router = server_builder.add_service(health_service); + + // Always add trading service (required) + router = router.add_service(TradingServiceServer::new(trading_proxy)); + + // Track service availability for logging + let backtesting_available = backtesting_proxy.is_some(); + let ml_training_available = ml_training_proxy.is_some(); + + // Conditionally add optional services + if let Some(backtesting) = backtesting_proxy { + router = router.add_service(BacktestingServiceServer::new(backtesting)); + } + if let Some(ml_training) = ml_training_proxy { + router = router.add_service(MlTrainingServiceServer::new(ml_training)); + } + + // Log startup information info!("🚀 API Gateway listening on {}", addr); - info!(" - Trading Service: {}", trading_backend_url); - info!(" - Backtesting Service: {}", backtesting_backend_url); - info!(" - ML Training Service: {}", ml_training_backend_url); + info!(" - Trading Service: {} (REQUIRED)", trading_backend_url); + info!(" - Backtesting Service: {} ({})", + backtesting_backend_url, + if backtesting_available { "✓ AVAILABLE" } else { "✗ UNAVAILABLE" }); + info!(" - ML Training Service: {} ({})", + ml_training_backend_url, + if ml_training_available { "✓ AVAILABLE" } else { "✗ UNAVAILABLE" }); info!(" - Health checks: enabled"); info!(" - Authentication: 6-layer (<10μs overhead)"); info!(" - Rate limiting: {} req/s per user", args.rate_limit_rps); + // Start server with graceful shutdown + let server = router.serve_with_shutdown(addr, async { + shutdown_rx.await.ok(); + }); + // Start server server.await?; diff --git a/services/backtesting_service/Cargo.toml b/services/backtesting_service/Cargo.toml index 7121246a8..e5cc56ff3 100644 --- a/services/backtesting_service/Cargo.toml +++ b/services/backtesting_service/Cargo.toml @@ -33,6 +33,9 @@ tonic.workspace = true tonic-prost.workspace = true prost.workspace = true +# HTTP server for health checks - USE WORKSPACE +axum.workspace = true + # Database - USE WORKSPACE sqlx.workspace = true influxdb2.workspace = true diff --git a/services/backtesting_service/Dockerfile b/services/backtesting_service/Dockerfile index 918e195d0..57099bf4f 100644 --- a/services/backtesting_service/Dockerfile +++ b/services/backtesting_service/Dockerfile @@ -89,12 +89,12 @@ RUN chmod +x ./backtesting_service # Switch to non-root user USER foxhunt -# Expose gRPC and metrics ports -EXPOSE 50052 9093 +# Expose gRPC, metrics, and health check ports +EXPOSE 50052 9093 8080 -# Health check using grpc_health_probe +# Health check using HTTP endpoint (doesn't require TLS) HEALTHCHECK --interval=10s --timeout=5s --start-period=30s --retries=3 \ - CMD /usr/local/bin/grpc_health_probe -addr=localhost:50052 || exit 1 + CMD curl -f http://localhost:8080/health || exit 1 # Run the application ENTRYPOINT ["./backtesting_service"] diff --git a/services/backtesting_service/src/health.rs b/services/backtesting_service/src/health.rs new file mode 100644 index 000000000..858409184 --- /dev/null +++ b/services/backtesting_service/src/health.rs @@ -0,0 +1,45 @@ +//! Health check endpoints for backtesting service +//! +//! Provides HTTP-based health and readiness endpoints that don't require TLS, +//! allowing Docker Compose health checks to work without client certificates. + +use axum::{routing::get, Json, Router}; +use serde::Serialize; + +/// Health status response +#[derive(Serialize)] +pub struct HealthResponse { + /// Service status + pub status: &'static str, + /// Service name + pub service: &'static str, + /// Service version + pub version: &'static str, +} + +/// Creates the health check router with /health and /ready endpoints +pub fn health_router() -> Router { + Router::new() + .route("/health", get(health_check)) + .route("/ready", get(readiness_check)) +} + +/// Liveness probe - indicates service is running +async fn health_check() -> Json { + Json(HealthResponse { + status: "healthy", + service: "backtesting", + version: env!("CARGO_PKG_VERSION"), + }) +} + +/// Readiness probe - indicates service is ready to accept requests +async fn readiness_check() -> Json { + // In the future, can add database connection checks here + // For now, if the service is running, it's ready + Json(HealthResponse { + status: "ready", + service: "backtesting", + version: env!("CARGO_PKG_VERSION"), + }) +} diff --git a/services/backtesting_service/src/main.rs b/services/backtesting_service/src/main.rs index cf67205da..4b66db39a 100644 --- a/services/backtesting_service/src/main.rs +++ b/services/backtesting_service/src/main.rs @@ -14,6 +14,7 @@ use tonic::transport::Server; use tracing::{info, warn}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; +mod health; mod performance; mod repositories; mod repository_impl; @@ -190,6 +191,29 @@ async fn main() -> Result<()> { .max_concurrent_streams(Some(10_000)); // Increased from 1,024 to 10,000 for production scale } + // Start HTTP health check server on port 8080 (no TLS required) + let health_port = std::env::var("HEALTH_PORT") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(8080); + let health_addr: SocketAddr = format!("0.0.0.0:{}", health_port) + .parse() + .context("Invalid health server address")?; + + info!("Starting health check server on {}", health_addr); + + let health_app = health::health_router(); + + tokio::spawn(async move { + let health_listener = tokio::net::TcpListener::bind(health_addr) + .await + .expect("Failed to bind health server"); + + axum::serve(health_listener, health_app) + .await + .expect("Health server failed"); + }); + // Build server with TLS and HTTP/2 optimizations server_builder .tls_config(tls_config.to_server_tls_config())? diff --git a/services/ml_training_service/Dockerfile.cpu b/services/ml_training_service/Dockerfile.cpu new file mode 100644 index 000000000..1bf98ede2 --- /dev/null +++ b/services/ml_training_service/Dockerfile.cpu @@ -0,0 +1,107 @@ +# CPU-only ML Training Service Dockerfile +# Fallback for environments without NVIDIA Container Runtime + +# ============================================================================= +# Builder Stage +# ============================================================================= +FROM rust:1.83-slim AS builder + +# Install build dependencies +RUN apt-get update && apt-get install -y \ + build-essential \ + pkg-config \ + libssl-dev \ + protobuf-compiler \ + libpq-dev \ + ca-certificates \ + git \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Set working directory +WORKDIR /build + +# Enable git fetch via CLI for cargo +ENV CARGO_NET_GIT_FETCH_WITH_CLI=true + +# Copy workspace manifests +COPY Cargo.toml Cargo.lock ./ + +# Copy workspace members +COPY trading_engine ./trading_engine +COPY risk ./risk +COPY risk-data ./risk-data +COPY trading-data ./trading-data +COPY tli ./tli +COPY ml ./ml +COPY ml-data ./ml-data +COPY data ./data +COPY backtesting ./backtesting +COPY adaptive-strategy ./adaptive-strategy +COPY common ./common +COPY storage ./storage +COPY model_loader ./model_loader +COPY market-data ./market-data +COPY database ./database +COPY config ./config +COPY services/backtesting_service ./services/backtesting_service +COPY services/trading_service ./services/trading_service +COPY services/ml_training_service ./services/ml_training_service +COPY services/api_gateway ./services/api_gateway +COPY services/load_tests ./services/load_tests +COPY services/stress_tests ./services/stress_tests +COPY services/integration_tests ./services/integration_tests +COPY tests ./tests +COPY migrations ./migrations + +# Build without CUDA features +RUN cargo build --release \ + --package ml_training_service \ + --no-default-features + +# ============================================================================= +# Runtime Stage +# ============================================================================= +FROM debian:bookworm-slim + +# Install runtime dependencies +RUN apt-get update && apt-get install -y \ + ca-certificates \ + libssl3 \ + libpq5 \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Download and install grpc_health_probe +RUN curl -sSL https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/v0.4.25/grpc_health_probe-linux-amd64 \ + -o /usr/local/bin/grpc_health_probe && \ + chmod +x /usr/local/bin/grpc_health_probe + +# Create non-root user +RUN groupadd --system --gid 1000 foxhunt && \ + useradd --system --uid 1000 --gid foxhunt --shell /bin/bash foxhunt + +# Create application directories +RUN mkdir -p /app/config /app/logs /app/models /app/checkpoints && \ + chown -R foxhunt:foxhunt /app + +# Set working directory +WORKDIR /app + +# Copy binary from builder +COPY --from=builder /build/target/release/ml_training_service ./ml_training_service +RUN chmod +x ./ml_training_service + +# Switch to non-root user +USER foxhunt + +# Expose gRPC and metrics ports +EXPOSE 50053 9094 + +# Health check using grpc_health_probe +HEALTHCHECK --interval=10s --timeout=5s --start-period=30s --retries=3 \ + CMD /usr/local/bin/grpc_health_probe -addr=localhost:50053 || exit 1 + +# Run the application +ENTRYPOINT ["./ml_training_service"] +CMD ["serve"]