# nginx Load Balancer Configuration for Foxhunt HFT Trading System # Version: 1.0 # Last Updated: 2025-10-07 # # This configuration provides: # - Layer 7 (L7) gRPC load balancing # - TLS termination at load balancer # - Health checks using gRPC Health Checking Protocol # - Rate limiting by user tier # - DDoS protection # - Connection limits # ============================================================================ # UPSTREAM DEFINITIONS # ============================================================================ # API Gateway cluster (primary entry point for all clients) upstream api_gateway_backend { # Load balancing algorithm # - least_conn: Route to backend with fewest active connections (recommended for long-lived gRPC connections) # - round_robin: Default, simple rotation # - ip_hash: Session affinity based on client IP (use for stateful operations) least_conn; # Backend servers # Format: server : [parameters]; server api-gateway-1:50051 max_fails=3 fail_timeout=30s; server api-gateway-2:50051 max_fails=3 fail_timeout=30s; server api-gateway-3:50051 max_fails=3 fail_timeout=30s; # Health check (requires nginx Plus or nginx-module-health-check) # For open-source nginx, use passive health checks (max_fails) # For nginx Plus: # health_check interval=5s fails=2 passes=1 uri=/grpc.health.v1.Health/Check; # Keepalive connections to backend # Reuse connections instead of creating new ones (critical for gRPC performance) keepalive 100; keepalive_timeout 60s; } # Trading Service cluster (internal, routed via API Gateway) upstream trading_service_backend { least_conn; server trading-service-1:50052 max_fails=3 fail_timeout=30s; server trading-service-2:50052 max_fails=3 fail_timeout=30s; server trading-service-3:50052 max_fails=3 fail_timeout=30s; keepalive 100; } # Backtesting Service cluster (worker pool pattern) upstream backtesting_service_backend { least_conn; server backtesting-service-1:50053 max_fails=3 fail_timeout=30s; server backtesting-service-2:50053 max_fails=3 fail_timeout=30s; keepalive 50; } # ML Training Service cluster (GPU instances, fewer replicas) upstream ml_training_service_backend { least_conn; server ml-training-service-1:50054 max_fails=3 fail_timeout=30s; server ml-training-service-2:50054 max_fails=3 fail_timeout=30s; keepalive 20; } # ============================================================================ # RATE LIMITING ZONES # ============================================================================ # Rate limiting by user tier (extracted from JWT) # Zone size: 10m = 10 MB = ~160,000 IP addresses # Free tier: 100 requests/second limit_req_zone $jwt_user_tier zone=tier_free:10m rate=100r/s; # Premium tier: 500 requests/second limit_req_zone $jwt_user_tier zone=tier_premium:10m rate=500r/s; # Enterprise tier: 2000 requests/second limit_req_zone $jwt_user_tier zone=tier_enterprise:10m rate=2000r/s; # Internal services: No rate limit # (Identified by internal JWT or mTLS certificate) # Connection limits per IP (DDoS protection) limit_conn_zone $binary_remote_addr zone=addr:10m; # ============================================================================ # MAP DEFINITIONS # ============================================================================ # Extract user tier from JWT (requires lua-nginx-module or custom logic) # For simplicity, this example assumes tier is in a custom header # In production, extract from JWT claims map $http_x_user_tier $rate_limit_zone { default tier_free; "free" tier_free; "premium" tier_premium; "enterprise" tier_enterprise; "internal" ""; # No rate limit for internal services } # ============================================================================ # SERVER BLOCKS # ============================================================================ # HTTP server (redirect to HTTPS) server { listen 80; listen [::]:80; server_name foxhunt.trading; # Redirect all HTTP traffic to HTTPS return 301 https://$server_name$request_uri; } # HTTPS server (TLS termination + gRPC load balancing) server { # Listen on port 443 with HTTP/2 (required for gRPC) listen 443 ssl http2; listen [::]:443 ssl http2; server_name foxhunt.trading; # ======================================================================== # TLS CONFIGURATION # ======================================================================== # SSL certificate (replace with your actual certificate paths) ssl_certificate /etc/ssl/certs/foxhunt.crt; ssl_certificate_key /etc/ssl/private/foxhunt.key; # TLS protocols (TLS 1.2 and 1.3 only, no SSLv3/TLS 1.0/1.1) ssl_protocols TLSv1.2 TLSv1.3; # Cipher suites (prefer modern, secure ciphers) ssl_ciphers HIGH:!aNULL:!MD5:!3DES; ssl_prefer_server_ciphers on; # SSL session cache (improves performance by reusing TLS sessions) ssl_session_cache shared:SSL:10m; ssl_session_timeout 10m; # OCSP stapling (improves TLS handshake performance) ssl_stapling on; ssl_stapling_verify on; ssl_trusted_certificate /etc/ssl/certs/ca-bundle.crt; # HSTS (HTTP Strict Transport Security) add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; # ======================================================================== # GRPC CONFIGURATION # ======================================================================== # gRPC-specific settings grpc_read_timeout 300s; # 5 minutes (for long-running backtests) grpc_send_timeout 30s; grpc_connect_timeout 10s; # Buffer sizes (tune based on payload size) client_body_buffer_size 1M; client_max_body_size 10M; # HTTP/2 settings http2_max_concurrent_streams 128; http2_recv_timeout 300s; # ======================================================================== # RATE LIMITING & DDOS PROTECTION # ======================================================================== # Connection limit per IP (max 10 concurrent connections) limit_conn addr 10; # Rate limiting by user tier (applied per location) # See location blocks below # ======================================================================== # LOGGING # ======================================================================== # Access log (includes gRPC status) access_log /var/log/nginx/access.log combined; # Error log error_log /var/log/nginx/error.log warn; # ======================================================================== # LOCATIONS (ROUTING) # ======================================================================== # API Gateway (primary entry point for clients) location /foxhunt.api_gateway { # Apply rate limiting based on user tier limit_req zone=$rate_limit_zone burst=20 nodelay; # gRPC proxy to backend grpc_pass grpc://api_gateway_backend; # Pass original client IP (for logging and rate limiting) grpc_set_header X-Real-IP $remote_addr; grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for; grpc_set_header X-Forwarded-Proto $scheme; # Error handling grpc_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; grpc_next_upstream_tries 3; grpc_next_upstream_timeout 10s; } # Trading Service (internal, typically accessed via API Gateway) # Exposed for direct access if needed (e.g., for monitoring) location /foxhunt.trading_service { # Restrict to internal IPs only allow 10.0.0.0/8; # Internal network allow 172.16.0.0/12; # Docker network deny all; grpc_pass grpc://trading_service_backend; grpc_set_header X-Real-IP $remote_addr; grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } # Backtesting Service (internal) location /foxhunt.backtesting_service { allow 10.0.0.0/8; allow 172.16.0.0/12; deny all; grpc_pass grpc://backtesting_service_backend; grpc_set_header X-Real-IP $remote_addr; grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } # ML Training Service (internal) location /foxhunt.ml_training_service { allow 10.0.0.0/8; allow 172.16.0.0/12; deny all; grpc_pass grpc://ml_training_service_backend; grpc_set_header X-Real-IP $remote_addr; grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } # Health check endpoint (HTTP, not gRPC) location /health { access_log off; return 200 "healthy\n"; add_header Content-Type text/plain; } # Metrics endpoint (for Prometheus scraping) location /metrics { # Restrict to monitoring IPs only allow 10.0.0.0/8; deny all; # Stub status (requires --with-http_stub_status_module) stub_status; } # Deny all other requests location / { return 404; } } # ============================================================================ # MONITORING & OBSERVABILITY # ============================================================================ # Prometheus metrics exporter (requires nginx-prometheus-exporter) # Run separately as a sidecar container: # docker run -p 9113:9113 nginx/nginx-prometheus-exporter:latest \ # --nginx.scrape-uri=http://localhost:8080/metrics # Example Prometheus scrape config: # scrape_configs: # - job_name: 'nginx-lb' # static_configs: # - targets: ['nginx-lb:9113'] # ============================================================================ # NOTES # ============================================================================ # 1. Replace SSL certificate paths with your actual certificates: # - Production: Use Let's Encrypt (certbot) or corporate CA # - Development: Self-signed certificates (not recommended for production) # # 2. Adjust rate limits based on your traffic patterns: # - Monitor 429 (Too Many Requests) error rate in Prometheus # - Increase limits if legitimate users are throttled # # 3. Health checks: # - Open-source nginx: Uses passive health checks (max_fails) # - nginx Plus: Active health checks with /grpc.health.v1.Health/Check # - Alternative: Use external health checker (e.g., Kubernetes liveness probes) # # 4. Load balancing algorithms: # - least_conn: Best for gRPC (long-lived connections) # - round_robin: Simpler, but can lead to uneven load # - ip_hash: Use only if session affinity is required # # 5. Monitoring: # - Use nginx-prometheus-exporter for metrics # - Monitor: request rate, error rate, latency, active connections # - Alerts: high error rate (> 1%), high latency (P99 > 100ms) # # 6. Scaling: # - Add more backend servers to upstream blocks # - Reload nginx config: nginx -s reload (zero-downtime) # - For Kubernetes: Use Ingress controller (nginx-ingress) # ============================================================================ # PRODUCTION CHECKLIST # ============================================================================ # [ ] Replace SSL certificates with production certificates # [ ] Configure DNS for foxhunt.trading # [ ] Set up log rotation (logrotate) # [ ] Enable Prometheus metrics exporter # [ ] Configure alerts (high error rate, high latency) # [ ] Test rate limiting with load testing tools # [ ] Configure firewall rules (allow 80, 443; deny others) # [ ] Enable nginx status page (for debugging) # [ ] Document backend server IP addresses # [ ] Set up automated certificate renewal (certbot cron)