foxhunt/config/nginx-lb.conf

# nginx Load Balancer Configuration for Foxhunt HFT Trading System
# Version: 1.0
# Last Updated: 2025-10-07
#
# This configuration provides:
# - Layer 7 (L7) gRPC load balancing
# - TLS termination at load balancer
# - Health checks using gRPC Health Checking Protocol
# - Rate limiting by user tier
# - DDoS protection
# - Connection limits

# ============================================================================
# UPSTREAM DEFINITIONS
# ============================================================================

# API Gateway cluster (primary entry point for all clients)
upstream api_gateway_backend {
    # Load balancing algorithm
    # - least_conn: Route to backend with fewest active connections (recommended for long-lived gRPC connections)
    # - round_robin: Default, simple rotation
    # - ip_hash: Session affinity based on client IP (use for stateful operations)
    least_conn;

    # Backend servers
    # Format: server <host>:<port> [parameters];
    server api-gateway-1:50051 max_fails=3 fail_timeout=30s;
    server api-gateway-2:50051 max_fails=3 fail_timeout=30s;
    server api-gateway-3:50051 max_fails=3 fail_timeout=30s;

    # Health check (requires nginx Plus or nginx-module-health-check)
    # For open-source nginx, use passive health checks (max_fails)
    # For nginx Plus:
    # health_check interval=5s fails=2 passes=1 uri=/grpc.health.v1.Health/Check;

    # Keepalive connections to backend
    # Reuse connections instead of creating new ones (critical for gRPC performance)
    keepalive 100;
    keepalive_timeout 60s;
}

# Trading Service cluster (internal, routed via API Gateway)
upstream trading_service_backend {
    least_conn;
    server trading-service-1:50052 max_fails=3 fail_timeout=30s;
    server trading-service-2:50052 max_fails=3 fail_timeout=30s;
    server trading-service-3:50052 max_fails=3 fail_timeout=30s;
    keepalive 100;
}

# Backtesting Service cluster (worker pool pattern)
upstream backtesting_service_backend {
    least_conn;
    server backtesting-service-1:50053 max_fails=3 fail_timeout=30s;
    server backtesting-service-2:50053 max_fails=3 fail_timeout=30s;
    keepalive 50;
}

# ML Training Service cluster (GPU instances, fewer replicas)
upstream ml_training_service_backend {
    least_conn;
    server ml-training-service-1:50054 max_fails=3 fail_timeout=30s;
    server ml-training-service-2:50054 max_fails=3 fail_timeout=30s;
    keepalive 20;
}

# ============================================================================
# RATE LIMITING ZONES
# ============================================================================

# Rate limiting by user tier (extracted from JWT)
# Zone size: 10m = 10 MB = ~160,000 IP addresses

# Free tier: 100 requests/second
limit_req_zone $jwt_user_tier zone=tier_free:10m rate=100r/s;

# Premium tier: 500 requests/second
limit_req_zone $jwt_user_tier zone=tier_premium:10m rate=500r/s;

# Enterprise tier: 2000 requests/second
limit_req_zone $jwt_user_tier zone=tier_enterprise:10m rate=2000r/s;

# Internal services: No rate limit
# (Identified by internal JWT or mTLS certificate)

# Connection limits per IP (DDoS protection)
limit_conn_zone $binary_remote_addr zone=addr:10m;

# ============================================================================
# MAP DEFINITIONS
# ============================================================================

# Extract user tier from JWT (requires lua-nginx-module or custom logic)
# For simplicity, this example assumes tier is in a custom header
# In production, extract from JWT claims
map $http_x_user_tier $rate_limit_zone {
    default         tier_free;
    "free"          tier_free;
    "premium"       tier_premium;
    "enterprise"    tier_enterprise;
    "internal"      "";  # No rate limit for internal services
}

# ============================================================================
# SERVER BLOCKS
# ============================================================================

# HTTP server (redirect to HTTPS)
server {
    listen 80;
    listen [::]:80;
    server_name foxhunt.trading;

    # Redirect all HTTP traffic to HTTPS
    return 301 https://$server_name$request_uri;
}

# HTTPS server (TLS termination + gRPC load balancing)
server {
    # Listen on port 443 with HTTP/2 (required for gRPC)
    listen 443 ssl http2;
    listen [::]:443 ssl http2;

    server_name foxhunt.trading;

    # ========================================================================
    # TLS CONFIGURATION
    # ========================================================================

    # SSL certificate (replace with your actual certificate paths)
    ssl_certificate /etc/ssl/certs/foxhunt.crt;
    ssl_certificate_key /etc/ssl/private/foxhunt.key;

    # TLS protocols (TLS 1.2 and 1.3 only, no SSLv3/TLS 1.0/1.1)
    ssl_protocols TLSv1.2 TLSv1.3;

    # Cipher suites (prefer modern, secure ciphers)
    ssl_ciphers HIGH:!aNULL:!MD5:!3DES;
    ssl_prefer_server_ciphers on;

    # SSL session cache (improves performance by reusing TLS sessions)
    ssl_session_cache shared:SSL:10m;
    ssl_session_timeout 10m;

    # OCSP stapling (improves TLS handshake performance)
    ssl_stapling on;
    ssl_stapling_verify on;
    ssl_trusted_certificate /etc/ssl/certs/ca-bundle.crt;

    # HSTS (HTTP Strict Transport Security)
    add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;

    # ========================================================================
    # GRPC CONFIGURATION
    # ========================================================================

    # gRPC-specific settings
    grpc_read_timeout 300s;  # 5 minutes (for long-running backtests)
    grpc_send_timeout 30s;
    grpc_connect_timeout 10s;

    # Buffer sizes (tune based on payload size)
    client_body_buffer_size 1M;
    client_max_body_size 10M;

    # HTTP/2 settings
    http2_max_concurrent_streams 128;
    http2_recv_timeout 300s;

    # ========================================================================
    # RATE LIMITING & DDOS PROTECTION
    # ========================================================================

    # Connection limit per IP (max 10 concurrent connections)
    limit_conn addr 10;

    # Rate limiting by user tier (applied per location)
    # See location blocks below

    # ========================================================================
    # LOGGING
    # ========================================================================

    # Access log (includes gRPC status)
    access_log /var/log/nginx/access.log combined;

    # Error log
    error_log /var/log/nginx/error.log warn;

    # ========================================================================
    # LOCATIONS (ROUTING)
    # ========================================================================

    # API Gateway (primary entry point for clients)
    location /foxhunt.api_gateway {
        # Apply rate limiting based on user tier
        limit_req zone=$rate_limit_zone burst=20 nodelay;

        # gRPC proxy to backend
        grpc_pass grpc://api_gateway_backend;

        # Pass original client IP (for logging and rate limiting)
        grpc_set_header X-Real-IP $remote_addr;
        grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        grpc_set_header X-Forwarded-Proto $scheme;

        # Error handling
        grpc_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
        grpc_next_upstream_tries 3;
        grpc_next_upstream_timeout 10s;
    }

    # Trading Service (internal, typically accessed via API Gateway)
    # Exposed for direct access if needed (e.g., for monitoring)
    location /foxhunt.trading_service {
        # Restrict to internal IPs only
        allow 10.0.0.0/8;     # Internal network
        allow 172.16.0.0/12;  # Docker network
        deny all;

        grpc_pass grpc://trading_service_backend;
        grpc_set_header X-Real-IP $remote_addr;
        grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    }

    # Backtesting Service (internal)
    location /foxhunt.backtesting_service {
        allow 10.0.0.0/8;
        allow 172.16.0.0/12;
        deny all;

        grpc_pass grpc://backtesting_service_backend;
        grpc_set_header X-Real-IP $remote_addr;
        grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    }

    # ML Training Service (internal)
    location /foxhunt.ml_training_service {
        allow 10.0.0.0/8;
        allow 172.16.0.0/12;
        deny all;

        grpc_pass grpc://ml_training_service_backend;
        grpc_set_header X-Real-IP $remote_addr;
        grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    }

    # Health check endpoint (HTTP, not gRPC)
    location /health {
        access_log off;
        return 200 "healthy\n";
        add_header Content-Type text/plain;
    }

    # Metrics endpoint (for Prometheus scraping)
    location /metrics {
        # Restrict to monitoring IPs only
        allow 10.0.0.0/8;
        deny all;

        # Stub status (requires --with-http_stub_status_module)
        stub_status;
    }

    # Deny all other requests
    location / {
        return 404;
    }
}

# ============================================================================
# MONITORING & OBSERVABILITY
# ============================================================================

# Prometheus metrics exporter (requires nginx-prometheus-exporter)
# Run separately as a sidecar container:
# docker run -p 9113:9113 nginx/nginx-prometheus-exporter:latest \
#   --nginx.scrape-uri=http://localhost:8080/metrics

# Example Prometheus scrape config:
# scrape_configs:
#   - job_name: 'nginx-lb'
#     static_configs:
#       - targets: ['nginx-lb:9113']

# ============================================================================
# NOTES
# ============================================================================

# 1. Replace SSL certificate paths with your actual certificates:
#    - Production: Use Let's Encrypt (certbot) or corporate CA
#    - Development: Self-signed certificates (not recommended for production)
#
# 2. Adjust rate limits based on your traffic patterns:
#    - Monitor 429 (Too Many Requests) error rate in Prometheus
#    - Increase limits if legitimate users are throttled
#
# 3. Health checks:
#    - Open-source nginx: Uses passive health checks (max_fails)
#    - nginx Plus: Active health checks with /grpc.health.v1.Health/Check
#    - Alternative: Use external health checker (e.g., Kubernetes liveness probes)
#
# 4. Load balancing algorithms:
#    - least_conn: Best for gRPC (long-lived connections)
#    - round_robin: Simpler, but can lead to uneven load
#    - ip_hash: Use only if session affinity is required
#
# 5. Monitoring:
#    - Use nginx-prometheus-exporter for metrics
#    - Monitor: request rate, error rate, latency, active connections
#    - Alerts: high error rate (> 1%), high latency (P99 > 100ms)
#
# 6. Scaling:
#    - Add more backend servers to upstream blocks
#    - Reload nginx config: nginx -s reload (zero-downtime)
#    - For Kubernetes: Use Ingress controller (nginx-ingress)

# ============================================================================
# PRODUCTION CHECKLIST
# ============================================================================

# [ ] Replace SSL certificates with production certificates
# [ ] Configure DNS for foxhunt.trading
# [ ] Set up log rotation (logrotate)
# [ ] Enable Prometheus metrics exporter
# [ ] Configure alerts (high error rate, high latency)
# [ ] Test rate limiting with load testing tools
# [ ] Configure firewall rules (allow 80, 443; deny others)
# [ ] Enable nginx status page (for debugging)
# [ ] Document backend server IP addresses
# [ ] Set up automated certificate renewal (certbot cron)