foxhunt/config/haproxy-lb.cfg

# HAProxy Load Balancer Configuration for Foxhunt HFT Trading System
# Version: 1.0
# Last Updated: 2025-10-07
#
# This configuration provides:
# - Layer 7 (L7) gRPC load balancing
# - TLS termination at load balancer
# - Advanced health checks (gRPC Health Checking Protocol)
# - Rate limiting (via stick tables)
# - High availability with stats page
# - Connection limits

# ============================================================================
# GLOBAL SETTINGS
# ============================================================================

global
    # Process management
    daemon
    maxconn 4096  # Maximum concurrent connections

    # Logging
    log /dev/log local0
    log /dev/log local1 notice

    # User/group for HAProxy process
    user haproxy
    group haproxy

    # Security
    chroot /var/lib/haproxy
    pidfile /var/run/haproxy.pid

    # SSL/TLS settings
    ssl-default-bind-ciphers ECDHE+AESGCM:ECDHE+CHACHA20:!aNULL:!MD5:!DSS
    ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets
    ssl-default-server-ciphers ECDHE+AESGCM:ECDHE+CHACHA20:!aNULL:!MD5:!DSS
    ssl-default-server-options ssl-min-ver TLSv1.2 no-tls-tickets

    # Performance tuning
    tune.ssl.default-dh-param 2048
    tune.h2.initial-window-size 65536
    tune.h2.max-concurrent-streams 128

# ============================================================================
# DEFAULTS
# ============================================================================

defaults
    # Mode (tcp for raw TCP, http for HTTP/gRPC)
    mode http

    # Logging
    log global
    option httplog
    option dontlognull

    # Timeouts
    timeout connect 10s     # Time to establish connection to backend
    timeout client 300s     # Client inactivity timeout (5 minutes for long-running gRPC)
    timeout server 300s     # Server inactivity timeout
    timeout http-request 10s
    timeout http-keep-alive 10s

    # Health checks
    timeout check 5s

    # Error handling
    retries 3
    option redispatch

    # HTTP/2 support (required for gRPC)
    option http-use-htx  # Enable HTTP/2

# ============================================================================
# STATS PAGE (Monitoring)
# ============================================================================

listen stats
    bind *:8404
    mode http
    stats enable
    stats uri /stats
    stats refresh 10s
    stats show-legends
    stats show-node

    # Authentication (change username/password in production)
    stats auth admin:foxhunt123

    # Additional stats options
    stats admin if TRUE  # Enable admin commands (drain, disable servers)

# ============================================================================
# FRONTEND (External Traffic)
# ============================================================================

# HTTP Frontend (redirect to HTTPS)
frontend http_frontend
    bind *:80
    mode http

    # Redirect all HTTP to HTTPS
    http-request redirect scheme https code 301

# HTTPS Frontend (TLS termination + gRPC load balancing)
frontend https_frontend
    # Bind to port 443 with TLS and HTTP/2 (ALPN: h2)
    bind *:443 ssl crt /etc/haproxy/certs/foxhunt.pem alpn h2,http/1.1

    mode http
    option httplog

    # Connection limits (DDoS protection)
    maxconn 2000

    # ========================================================================
    # ACCESS CONTROL LISTS (ACLs)
    # ========================================================================

    # Detect gRPC protocol (Content-Type: application/grpc)
    acl is_grpc hdr(content-type) -m beg application/grpc

    # Route based on gRPC service path
    acl is_api_gateway path_beg /foxhunt.api_gateway
    acl is_trading_service path_beg /foxhunt.trading_service
    acl is_backtesting_service path_beg /foxhunt.backtesting_service
    acl is_ml_training_service path_beg /foxhunt.ml_training_service

    # Internal IP ranges (for backend services)
    acl is_internal_ip src 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16

    # Health check endpoint
    acl is_health_check path /health

    # ========================================================================
    # RATE LIMITING (Stick Tables)
    # ========================================================================

    # Track request rate per client IP
    stick-table type ip size 100k expire 30s store http_req_rate(10s)

    # Track connection rate per client IP
    http-request track-sc0 src

    # Rate limit: 1000 requests per 10 seconds per IP
    http-request deny deny_status 429 if { sc_http_req_rate(0) gt 1000 }

    # Connection limit: 100 concurrent connections per IP
    http-request deny deny_status 429 if { src_conn_cur ge 100 }

    # ========================================================================
    # REQUEST HEADERS
    # ========================================================================

    # Add X-Forwarded-* headers (for backend logging)
    http-request set-header X-Real-IP %[src]
    http-request set-header X-Forwarded-For %[src]
    http-request set-header X-Forwarded-Proto https

    # ========================================================================
    # ROUTING
    # ========================================================================

    # Health check (return 200 OK)
    use_backend health_backend if is_health_check

    # Route gRPC traffic to appropriate backend
    use_backend api_gateway_backend if is_grpc is_api_gateway
    use_backend trading_service_backend if is_grpc is_trading_service is_internal_ip
    use_backend backtesting_service_backend if is_grpc is_backtesting_service is_internal_ip
    use_backend ml_training_service_backend if is_grpc is_ml_training_service is_internal_ip

    # Default: Return 404 for unknown routes
    default_backend not_found_backend

# ============================================================================
# BACKENDS (Service Clusters)
# ============================================================================

# API Gateway Backend (primary entry point)
backend api_gateway_backend
    mode http
    balance leastconn  # Route to backend with fewest connections (best for gRPC)

    # HTTP/2 support
    option http-use-htx

    # Health check using gRPC Health Checking Protocol
    option httpchk
    http-check send meth GET uri /grpc.health.v1.Health/Check ver HTTP/2
    http-check expect status 200

    # Backend servers
    # Format: server <name> <host>:<port> [options]
    server api-gateway-1 api-gateway-1:50051 check inter 5s rise 2 fall 3 maxconn 1000
    server api-gateway-2 api-gateway-2:50051 check inter 5s rise 2 fall 3 maxconn 1000
    server api-gateway-3 api-gateway-3:50051 check inter 5s rise 2 fall 3 maxconn 1000

    # Connection reuse (important for gRPC performance)
    http-reuse always

# Trading Service Backend
backend trading_service_backend
    mode http
    balance leastconn

    option http-use-htx

    # Health check
    option httpchk
    http-check send meth GET uri /grpc.health.v1.Health/Check ver HTTP/2
    http-check expect status 200

    # Backend servers
    server trading-service-1 trading-service-1:50052 check inter 5s rise 2 fall 3 maxconn 1000
    server trading-service-2 trading-service-2:50052 check inter 5s rise 2 fall 3 maxconn 1000
    server trading-service-3 trading-service-3:50052 check inter 5s rise 2 fall 3 maxconn 1000

    http-reuse always

# Backtesting Service Backend (worker pool pattern)
backend backtesting_service_backend
    mode http
    balance leastconn

    option http-use-htx

    # Health check
    option httpchk
    http-check send meth GET uri /grpc.health.v1.Health/Check ver HTTP/2
    http-check expect status 200

    # Backend servers (fewer instances, CPU-intensive workload)
    server backtesting-service-1 backtesting-service-1:50053 check inter 5s rise 2 fall 3 maxconn 500
    server backtesting-service-2 backtesting-service-2:50053 check inter 5s rise 2 fall 3 maxconn 500

    http-reuse always

# ML Training Service Backend (GPU instances)
backend ml_training_service_backend
    mode http
    balance leastconn

    option http-use-htx

    # Health check
    option httpchk
    http-check send meth GET uri /grpc.health.v1.Health/Check ver HTTP/2
    http-check expect status 200

    # Backend servers (GPU instances, fewer replicas)
    server ml-training-service-1 ml-training-service-1:50054 check inter 5s rise 2 fall 3 maxconn 200
    server ml-training-service-2 ml-training-service-2:50054 check inter 5s rise 2 fall 3 maxconn 200

    http-reuse always

# Health Check Backend (returns 200 OK)
backend health_backend
    mode http
    http-request return status 200 content-type "text/plain" string "healthy\n"

# Not Found Backend (returns 404)
backend not_found_backend
    mode http
    http-request return status 404 content-type "text/plain" string "not found\n"

# ============================================================================
# MONITORING & OBSERVABILITY
# ============================================================================

# HAProxy Stats (available at http://localhost:8404/stats)
# Metrics include:
# - Request rate, error rate, latency
# - Active connections per backend
# - Health check status
# - Session rate

# Prometheus Exporter:
# Use haproxy_exporter: https://github.com/prometheus/haproxy_exporter
# docker run -d -p 9101:9101 prom/haproxy-exporter:latest \
#   --haproxy.scrape-uri="http://admin:foxhunt123@localhost:8404/stats;csv"

# Example Prometheus scrape config:
# scrape_configs:
#   - job_name: 'haproxy'
#     static_configs:
#       - targets: ['haproxy:9101']

# ============================================================================
# NOTES
# ============================================================================

# 1. SSL Certificate:
#    - Combine certificate and key into single PEM file:
#      cat foxhunt.crt foxhunt.key > /etc/haproxy/certs/foxhunt.pem
#    - Ensure correct permissions: chmod 600 /etc/haproxy/certs/foxhunt.pem
#
# 2. Health Checks:
#    - Uses gRPC Health Checking Protocol (standard)
#    - Check interval: 5 seconds
#    - Rise: 2 successful checks = healthy
#    - Fall: 3 failed checks = unhealthy
#    - Backends must implement grpc.health.v1.Health service
#
# 3. Rate Limiting:
#    - Stick tables track request rate per IP
#    - Adjust limits based on traffic patterns
#    - Monitor 429 error rate to avoid over-throttling
#
# 4. Load Balancing Algorithm:
#    - leastconn: Best for gRPC (long-lived connections)
#    - Alternative: roundrobin (simpler, less accurate)
#    - Alternative: source (session affinity by IP)
#
# 5. Connection Reuse:
#    - http-reuse always: Reuse connections to backends (critical for gRPC)
#    - Reduces latency and connection overhead
#
# 6. Logging:
#    - Logs sent to syslog (/dev/log)
#    - Configure rsyslog to route HAProxy logs to file
#    - Example: /var/log/haproxy.log

# ============================================================================
# PRODUCTION CHECKLIST
# ============================================================================

# [ ] Replace SSL certificate path (/etc/haproxy/certs/foxhunt.pem)
# [ ] Change stats page credentials (admin:foxhunt123)
# [ ] Configure DNS for foxhunt.trading
# [ ] Set up log rotation (logrotate)
# [ ] Enable Prometheus exporter (haproxy_exporter)
# [ ] Configure alerts (high error rate, backend down)
# [ ] Test rate limiting with load testing tools
# [ ] Configure firewall rules (allow 80, 443, 8404; deny others)
# [ ] Document backend server IP addresses
# [ ] Test failover (kill backend, verify traffic reroutes)
# [ ] Set up automated certificate renewal (certbot cron)

# ============================================================================
# HAPROXY MANAGEMENT COMMANDS
# ============================================================================

# Start HAProxy:
#   haproxy -f /etc/haproxy/haproxy.cfg
#
# Check configuration:
#   haproxy -c -f /etc/haproxy/haproxy.cfg
#
# Reload configuration (zero-downtime):
#   haproxy -f /etc/haproxy/haproxy.cfg -sf $(cat /var/run/haproxy.pid)
#
# View stats:
#   http://localhost:8404/stats
#
# Admin commands (via stats page):
#   - Drain server: Set server to "drain" mode (no new connections)
#   - Disable server: Take server out of rotation
#   - Enable server: Put server back into rotation
#
# Socket commands (requires stats socket):
#   echo "show stat" | socat stdio /var/run/haproxy.sock
#   echo "show servers state" | socat stdio /var/run/haproxy.sock
#   echo "disable server api_gateway_backend/api-gateway-1" | socat stdio /var/run/haproxy.sock