diff --git a/Cargo.lock b/Cargo.lock index 59f96f1ad..ae3dfef9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9000,6 +9000,7 @@ dependencies = [ "hdrhistogram", "influxdb2", "jemalloc_pprof", + "jsonwebtoken", "lazy_static", "ml", "num 0.4.3", @@ -9010,6 +9011,7 @@ dependencies = [ "rand 0.8.5", "rand_distr 0.4.3", "redis", + "reqwest 0.12.23", "risk", "risk-data", "rstest 0.22.0", @@ -9027,6 +9029,7 @@ dependencies = [ "tokio-test", "toml", "tonic", + "tonic-health", "tracing", "tracing-subscriber", "trading_engine", diff --git a/EMERGENCY_PROCEDURES.md b/EMERGENCY_PROCEDURES.md new file mode 100644 index 000000000..a501f6b68 --- /dev/null +++ b/EMERGENCY_PROCEDURES.md @@ -0,0 +1,698 @@ +# Emergency Procedures - Foxhunt HFT Trading System + +**Purpose**: Critical incident response procedures for immediate action +**Version**: 1.0.0 +**Last Updated**: 2025-10-07 + +--- + +## 🚨 EMERGENCY CONTACTS + +**Fill in before production deployment:** + +| Role | Name | Phone | Email | Escalation Time | +|------|------|-------|-------|-----------------| +| On-Call Engineer | [CONFIGURE] | [CONFIGURE] | [CONFIGURE] | Immediate | +| Database Admin | [CONFIGURE] | [CONFIGURE] | [CONFIGURE] | 5 minutes | +| Security Lead | [CONFIGURE] | [CONFIGURE] | [CONFIGURE] | 10 minutes | +| CTO/Technical Lead | [CONFIGURE] | [CONFIGURE] | [CONFIGURE] | 15 minutes | +| CEO | [CONFIGURE] | [CONFIGURE] | [CONFIGURE] | 30 minutes | + +**PagerDuty**: [CONFIGURE] +**Slack Channel**: #foxhunt-incidents + +--- + +## 🎯 Severity Levels + +### SEV-1: CRITICAL (Immediate Response) +- **Impact**: Trading completely halted, data loss, security breach +- **Response Time**: <5 minutes +- **Examples**: System down, database corruption, unauthorized access +- **Escalation**: Immediate page to on-call + CTO + +### SEV-2: HIGH (Urgent Response) +- **Impact**: Significant degradation, trading impaired but operational +- **Response Time**: <15 minutes +- **Examples**: High latency (>500ΞΌs), service failure, data inconsistency +- **Escalation**: Page on-call, notify team in Slack + +### SEV-3: MEDIUM (Priority Response) +- **Impact**: Minor degradation, non-critical component failure +- **Response Time**: <1 hour +- **Examples**: Non-critical service down, elevated error rate +- **Escalation**: Slack notification, log ticket + +### SEV-4: LOW (Standard Response) +- **Impact**: No immediate impact, potential future issue +- **Response Time**: <4 hours (business hours) +- **Examples**: Warning alerts, resource usage trends +- **Escalation**: Standard ticket + +--- + +## πŸ”΄ CRITICAL SCENARIOS (SEV-1) + +### Scenario 1: System-Wide Trading Halt + +**Trigger**: All trading stopped, no orders processing +**Response Time**: <2 minutes to activate kill switch, <15 minutes to diagnose + +#### Immediate Actions (First 60 seconds) + +```bash +# 1. CONFIRM HALT - Check if trading actually stopped +curl http://localhost:9092/metrics | grep trading_active +# Expected: trading_active 0 + +# 2. ACTIVATE KILL SWITCH (if not already active) +export KILL_SWITCH_TOKEN="your-master-token" +curl -X POST http://localhost:8080/emergency/kill \ + -H "X-Kill-Switch-Token: $KILL_SWITCH_TOKEN" + +# 3. VERIFY KILL SWITCH ACTIVE +curl http://localhost:8080/status +# Expected: {"status": "HALTED", "reason": "Emergency kill switch activated"} + +# 4. NOTIFY STAKEHOLDERS +./deployment/scripts/send-emergency-alert.sh "Trading halted - investigating" +``` + +#### Diagnosis (Next 5 minutes) + +```bash +# Check service status +docker-compose ps # Docker +sudo systemctl status foxhunt-* # Bare-metal + +# Check recent errors +docker-compose logs --tail=100 trading_service | grep -i error +sudo journalctl -u foxhunt-trading --since "5 minutes ago" | grep -i error + +# Check system resources +df -h # Disk space +free -m # Memory +uptime # Load average + +# Check database connectivity +psql $DATABASE_URL -c "SELECT 1;" || echo "❌ Database connection failed" + +# Check network connectivity +ping -c 3 || echo "❌ Network connectivity failed" +``` + +#### Decision Tree + +**If database is down**: +```bash +# See: Scenario 2 - Database Failure +``` + +**If service crashed**: +```bash +# See: Scenario 3 - Service Crash +``` + +**If network issue**: +```bash +# See: Scenario 5 - Network Failure +``` + +**If cause unknown**: +```bash +# 1. Restart services in safe mode (read-only) +./deployment/scripts/restart-safe-mode.sh + +# 2. Escalate to CTO +./deployment/scripts/escalate.sh --level=cto --message="Trading halt, cause unknown" + +# 3. Begin detailed investigation +./deployment/scripts/capture-diagnostics.sh > /var/log/foxhunt/incident-$(date +%s).log +``` + +#### Recovery Procedure + +```bash +# 1. Fix root cause (see specific scenarios) + +# 2. Run validation tests +./deployment/scripts/pre-trading-validation.sh + +# 3. Resume trading (only after validation passes) +curl -X POST http://localhost:8080/trading/resume \ + -H "X-Kill-Switch-Token: $KILL_SWITCH_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"reason": "Issue resolved - verified", "approved_by": "your-name"}' + +# 4. Monitor closely for 30 minutes +watch -n 10 'curl -s http://localhost:9092/metrics | grep -E "(order_latency|error_rate)"' + +# 5. Send all-clear notification +./deployment/scripts/send-emergency-alert.sh "Trading resumed - incident resolved" +``` + +--- + +### Scenario 2: Database Failure + +**Trigger**: Cannot connect to PostgreSQL +**Response Time**: <5 minutes to failover + +#### Immediate Actions + +```bash +# 1. ACTIVATE KILL SWITCH +curl -X POST http://localhost:8080/emergency/kill \ + -H "X-Kill-Switch-Token: $KILL_SWITCH_TOKEN" + +# 2. CHECK DATABASE STATUS +sudo systemctl status postgresql # Bare-metal +docker-compose ps postgres # Docker + +# 3. CHECK REPLICATION STATUS (if configured) +psql -h replica-server -U foxhunt -c "SELECT pg_is_in_recovery();" +# Expected: t (true = replica) +``` + +#### Recovery Options + +**Option A: Database is Running but Unresponsive** +```bash +# 1. Check connections +psql $DATABASE_URL -c "SELECT count(*) FROM pg_stat_activity;" + +# 2. Kill long-running queries +psql $DATABASE_URL -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE state = 'active' AND now() - query_start > interval '5 minutes';" + +# 3. Restart PostgreSQL +sudo systemctl restart postgresql # Bare-metal +docker-compose restart postgres # Docker + +# 4. Wait for startup +sleep 10 + +# 5. Test connection +psql $DATABASE_URL -c "SELECT 1;" +``` + +**Option B: Database Crashed - Restore from Replica** +```bash +# 1. Promote replica to primary +ssh replica-server "sudo -u postgres /usr/lib/postgresql/16/bin/pg_ctl promote -D /var/lib/postgresql/16/main" + +# 2. Update connection strings +export NEW_DB_URL=postgresql://foxhunt:password@replica-server:5432/foxhunt + +# 3. Update services +docker-compose down +sed -i "s|DATABASE_URL=.*|DATABASE_URL=$NEW_DB_URL|" .env +docker-compose up -d + +# OR for bare-metal +sudo systemctl stop foxhunt-* +sudo systemctl set-environment DATABASE_URL="$NEW_DB_URL" +sudo systemctl start foxhunt-* + +# 4. Verify connectivity +psql $NEW_DB_URL -c "SELECT count(*) FROM orders;" +``` + +**Option C: Complete Restore from Backup** +```bash +# 1. Download latest backup from S3 +aws s3 cp s3://foxhunt-backups/postgres/latest.dump /tmp/restore.dump + +# 2. Create new database +createdb foxhunt_restored + +# 3. Restore +pg_restore -h localhost -U foxhunt -d foxhunt_restored -v /tmp/restore.dump + +# 4. Swap databases +psql -U postgres < | grep -E "(panic|error|fatal)" +sudo journalctl -u foxhunt- -n 100 | grep -E "(panic|error|fatal)" + +# 3. RESTART SERVICE +docker-compose restart # Docker +sudo systemctl restart foxhunt- # Bare-metal + +# 4. VERIFY RESTART SUCCESSFUL +docker-compose ps +sudo systemctl status foxhunt- + +# 5. CHECK FOR ERRORS AFTER RESTART +docker-compose logs --tail=50 +sudo journalctl -u foxhunt- -n 50 +``` + +#### If Restart Fails + +```bash +# 1. Check for port conflicts +sudo lsof -i :50052 # Trading service port +sudo kill -9 # Kill conflicting process + +# 2. Check disk space +df -h +# If >90% full, clean up logs +sudo find /var/log/foxhunt -name "*.log" -mtime +7 -delete + +# 3. Check memory +free -m +# If memory exhausted, identify memory hog +ps aux --sort=-%mem | head -10 + +# 4. Try safe mode restart +docker-compose stop +docker-compose up -d --force-recreate + +# OR bare-metal +sudo systemctl stop foxhunt- +sudo rm -rf /tmp/foxhunt/* # Clear temp files +sudo systemctl start foxhunt- + +# 5. If still failing, rollback +sudo cp /opt/foxhunt/bin/.backup /opt/foxhunt/bin/ +sudo systemctl restart foxhunt- +``` + +--- + +### Scenario 4: Security Breach Detected + +**Trigger**: Unauthorized access, data exfiltration, suspicious activity +**Response Time**: IMMEDIATE - <30 seconds to isolate + +#### Immediate Actions (CRITICAL - Do NOT delay) + +```bash +# 1. ACTIVATE KILL SWITCH IMMEDIATELY +curl -X POST http://localhost:8080/emergency/kill \ + -H "X-Kill-Switch-Token: $KILL_SWITCH_TOKEN" + +# 2. ISOLATE SYSTEM (disconnect from network) +# Bare-metal +sudo iptables -P INPUT DROP +sudo iptables -P OUTPUT DROP +sudo iptables -P FORWARD DROP + +# Docker +docker network disconnect foxhunt_foxhunt-network + +# 3. CAPTURE FORENSIC DATA +./deployment/scripts/capture-forensics.sh > /var/log/foxhunt/security-incident-$(date +%s).log + +# 4. NOTIFY SECURITY TEAM IMMEDIATELY +./deployment/scripts/security-alert.sh --severity=critical --message="Security breach detected - system isolated" + +# 5. PRESERVE EVIDENCE +sudo cp -r /var/log/foxhunt /var/log/foxhunt.incident-$(date +%s) +sudo tar -czf /tmp/incident-evidence-$(date +%s).tar.gz /var/log/foxhunt /opt/foxhunt/config +``` + +#### Investigation (Security Team) + +```bash +# 1. Review audit logs +grep "$(date +%Y-%m-%d)" /var/log/foxhunt/audit.log | grep -E "(unauthorized|failed|suspicious)" + +# 2. Check for unauthorized access +psql $DATABASE_URL -c "SELECT * FROM audit_logs WHERE action = 'login_failed' AND created_at > NOW() - INTERVAL '24 hours' ORDER BY created_at DESC;" + +# 3. Review network connections +sudo netstat -anp | grep ESTABLISHED + +# 4. Check for malware +sudo rkhunter --check +sudo chkrootkit + +# 5. Check file integrity +sudo aide --check + +# 6. Review user access +sudo lastlog +sudo last -n 50 +``` + +#### Recovery (After Investigation) + +```bash +# 1. Rotate ALL secrets +vault kv put secret/foxhunt/jwt secret="$(openssl rand -base64 64)" +vault kv put secret/foxhunt/kill_switch master_token="$(openssl rand -base64 32)" +# ... rotate all other secrets + +# 2. Update firewall rules +sudo ufw default deny incoming +sudo ufw allow from to any port 50051 + +# 3. Apply security patches +sudo apt-get update +sudo apt-get upgrade -y + +# 4. Restore network access (carefully) +sudo iptables -F +sudo systemctl restart foxhunt-* + +# 5. Monitor closely for 48 hours +watch -n 60 './deployment/scripts/security-scan.sh' +``` + +--- + +### Scenario 5: Network Failure/Partition + +**Trigger**: Cannot reach exchange servers, inter-service communication failed +**Response Time**: <10 seconds to halt trading + +#### Immediate Actions + +```bash +# 1. HALT TRADING (network issues = blind trading) +curl -X POST http://localhost:8080/emergency/kill \ + -H "X-Kill-Switch-Token: $KILL_SWITCH_TOKEN" + +# 2. VERIFY NETWORK CONNECTIVITY +# Check external connectivity +ping -c 3 8.8.8.8 # Internet +ping -c 3 # Exchange + +# Check inter-service connectivity +for service in trading backtesting ml_training; do + curl -s http://$service:9092/metrics || echo "❌ Cannot reach $service" +done + +# 3. CHECK NETWORK INTERFACE STATUS +ip link show +ip addr show + +# 4. CHECK FIREWALL RULES +sudo iptables -L -n -v +``` + +#### Recovery Options + +**Option A: Network Restored Automatically** +```bash +# Services will auto-recover when network returns +# Verify connectivity restored +ping -c 10 + +# Services should exit read-only mode automatically +curl http://localhost:9092/metrics | grep network_partition_active +# Expected: network_partition_active 0 + +# Resume trading after verification +curl -X POST http://localhost:8080/trading/resume \ + -H "X-Kill-Switch-Token: $KILL_SWITCH_TOKEN" +``` + +**Option B: Manual Network Recovery** +```bash +# 1. Restart network interface +sudo ifdown eth0 +sudo ifup eth0 + +# 2. Restart network service +sudo systemctl restart networking + +# 3. Verify connectivity +ping -c 5 + +# 4. Restart services +sudo systemctl restart foxhunt-* + +# 5. Resume trading +curl -X POST http://localhost:8080/trading/resume \ + -H "X-Kill-Switch-Token: $KILL_SWITCH_TOKEN" +``` + +--- + +### Scenario 6: High Latency Alert (>100ΞΌs p99) + +**Trigger**: Order processing latency exceeds critical threshold +**Response Time**: <5 minutes to investigate + +#### Immediate Actions + +```bash +# 1. VERIFY LATENCY SPIKE +curl http://localhost:9092/metrics | grep order_processing_duration_seconds + +# 2. CHECK CPU FREQUENCY SCALING (should be "performance") +cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor + +# 3. CHECK SYSTEM LOAD +uptime +top -bn1 | head -20 + +# 4. CHECK DATABASE QUERY PERFORMANCE +psql $DATABASE_URL -c "SELECT query, mean_exec_time, calls FROM pg_stat_statements ORDER BY mean_exec_time DESC LIMIT 10;" + +# 5. CHECK NETWORK LATENCY +ping -c 10 +``` + +#### Recovery Actions + +**Fix CPU Frequency Scaling**: +```bash +sudo cpupower frequency-set -g performance +``` + +**Fix Database Performance**: +```bash +# Add missing indexes +psql $DATABASE_URL <90%) + +```bash +# 1. CHECK DISK USAGE +df -h + +# 2. FIND LARGE FILES +sudo du -ah /var/log/foxhunt | sort -rh | head -20 +sudo du -ah /opt/foxhunt | sort -rh | head -20 + +# 3. CLEAN UP LOGS (keep last 7 days) +sudo find /var/log/foxhunt -name "*.log" -mtime +7 -delete +sudo journalctl --vacuum-time=7d + +# 4. CLEAN UP DOCKER (if using Docker) +docker system prune -af --volumes + +# 5. EXPAND DISK (if possible) +sudo lvextend -L +100G /dev/vg0/root +sudo resize2fs /dev/vg0/root +``` + +### Scenario 8: Memory Exhaustion (>90%) + +```bash +# 1. IDENTIFY MEMORY HOGS +ps aux --sort=-%mem | head -10 + +# 2. RESTART SERVICES (frees memory) +sudo systemctl restart foxhunt-trading + +# 3. INCREASE SWAP (temporary fix) +sudo fallocate -l 8G /swapfile +sudo chmod 600 /swapfile +sudo mkswap /swapfile +sudo swapon /swapfile + +# 4. INCREASE SERVICE MEMORY LIMITS +sudo systemctl edit foxhunt-trading +# Add: MemoryMax=4G +sudo systemctl restart foxhunt-trading + +# 5. SCALE HORIZONTALLY (long-term fix) +kubectl scale deployment trading-service --replicas=5 +``` + +### Scenario 9: Redis Connection Failure + +```bash +# 1. CHECK REDIS STATUS +redis-cli ping +sudo systemctl status redis + +# 2. RESTART REDIS +sudo systemctl restart redis # Bare-metal +docker-compose restart redis # Docker + +# 3. VERIFY CONNECTION FROM SERVICES +redis-cli -h redis ping + +# 4. CHECK CONNECTION POOL +redis-cli info clients + +# 5. RESTART SERVICES (reconnect to Redis) +sudo systemctl restart foxhunt-* +``` + +--- + +## πŸ“ž Escalation Procedures + +### Escalation Matrix + +| Time Elapsed | Action | +|--------------|--------| +| T+0 | On-call engineer paged | +| T+5 min | If no response, escalate to backup on-call | +| T+10 min | Notify CTO/Technical Lead | +| T+15 min | If SEV-1, notify CEO | +| T+30 min | If unresolved, engage external support | + +### Communication Protocol + +**Internal (Slack #foxhunt-incidents)**: +``` +🚨 INCIDENT ALERT +Severity: SEV-X +Component: +Impact: +Detected: +On-Call: responding +ETA: +``` + +**External (Stakeholders)**: +``` +Subject: [INCIDENT] Foxhunt Trading System - + +Time: +Status: Investigating/Identified/Resolved +Impact: +Actions: +ETA: +Contact: +``` + +--- + +## πŸ“Š Post-Incident Procedures + +### Immediate (Within 1 hour of resolution) + +```bash +# 1. Document incident +cat > /var/log/foxhunt/incidents/incident-$(date +%Y%m%d-%H%M%S).md < +**Root Cause**: +**Resolution**: +**Responders**: +EOF + +# 2. Send all-clear notification +./deployment/scripts/send-alert.sh "Incident resolved - system operational" + +# 3. Monitor for recurrence +watch -n 60 './deployment/health_check.sh --mode comprehensive' +``` + +### Post-Mortem (Within 48 hours) + +1. **Incident Timeline**: Document minute-by-minute actions +2. **Root Cause Analysis**: 5 Whys methodology +3. **Action Items**: Prevent recurrence +4. **Communication Review**: How well did we communicate? +5. **Documentation Updates**: Update runbooks with lessons learned + +--- + +## πŸ”§ Emergency Tools + +### Quick Diagnostic Script + +```bash +#!/bin/bash +# /opt/foxhunt/bin/emergency-diagnostics.sh + +echo "=== FOXHUNT EMERGENCY DIAGNOSTICS ===" +echo "Time: $(date)" +echo "" + +echo "=== SERVICE STATUS ===" +systemctl status foxhunt-* --no-pager | grep -E "(Active|Main PID)" + +echo "" +echo "=== SYSTEM RESOURCES ===" +echo "Load: $(uptime | awk -F'load average:' '{print $2}')" +echo "Memory: $(free -m | grep Mem | awk '{printf "%.1f%%", $3/$2*100}')" +echo "Disk: $(df -h / | tail -1 | awk '{print $5}')" + +echo "" +echo "=== DATABASE ===" +psql $DATABASE_URL -c "SELECT 1;" && echo "βœ… Connected" || echo "❌ Failed" + +echo "" +echo "=== RECENT ERRORS ===" +journalctl -u foxhunt-* --since "5 minutes ago" | grep -i error | tail -10 + +echo "" +echo "=== NETWORK ===" +ping -c 3 8.8.8.8 && echo "βœ… Internet OK" || echo "❌ Internet Failed" +``` + +--- + +**Last Updated**: 2025-10-07 +**Version**: 1.0.0 +**Review Frequency**: Monthly or after each incident diff --git a/LOAD_BALANCING_SCALING.md b/LOAD_BALANCING_SCALING.md new file mode 100644 index 000000000..2e0fb7c06 --- /dev/null +++ b/LOAD_BALANCING_SCALING.md @@ -0,0 +1,1283 @@ +# Load Balancing & Horizontal Scaling Guide + +**Last Updated**: 2025-10-07 +**Status**: Production Ready +**Audience**: DevOps, SRE, Platform Engineers + +--- + +## Table of Contents + +1. [Architecture Overview](#architecture-overview) +2. [Load Balancer Architecture](#load-balancer-architecture) +3. [API Gateway Load Balancing](#api-gateway-load-balancing) +4. [Backend Service Scaling](#backend-service-scaling) +5. [Horizontal Pod Autoscaling (HPA)](#horizontal-pod-autoscaling-hpa) +6. [Database Scaling](#database-scaling) +7. [Redis Scaling](#redis-scaling) +8. [Performance Targets](#performance-targets) +9. [Cost Optimization](#cost-optimization) +10. [Monitoring & Alerting](#monitoring--alerting) + +--- + +## Architecture Overview + +### Scaling Topology + +``` + Internet + | + v + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ Load Balancer β”‚ (nginx/HAProxy/AWS ALB) + β”‚ TLS Terminationβ”‚ + β”‚ DDoS Protectionβ”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + | + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + v v v + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ Gateway β”‚ β”‚ Gateway β”‚ β”‚ Gateway β”‚ + β”‚ Pod 1 β”‚ β”‚ Pod 2 β”‚ β”‚ Pod N β”‚ + β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”˜ + | | | + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + | + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + v v v + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ Trading β”‚ β”‚Backtestingβ”‚ β”‚ ML Train β”‚ + β”‚ Service β”‚ β”‚ Service β”‚ β”‚ Service β”‚ + β”‚ (1-N) β”‚ β”‚ (1-M) β”‚ β”‚ (1-K) β”‚ + β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”˜ + | | | + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + | + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + v v v + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚PostgreSQLβ”‚ β”‚ Redis β”‚ β”‚InfluxDB β”‚ + β”‚(Primary +β”‚ β”‚ Cluster β”‚ β”‚ Cluster β”‚ + β”‚ Replicas)β”‚ β”‚ β”‚ β”‚ β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +### Key Principles + +1. **Stateless Services**: All application services are stateless for horizontal scaling +2. **External State**: State stored in PostgreSQL, Redis, or S3 +3. **Health-Based Routing**: Load balancer only routes to healthy instances +4. **Graceful Degradation**: System continues with reduced capacity during failures +5. **Auto-Scaling**: Automatic scaling based on metrics (CPU, memory, request rate) + +--- + +## Load Balancer Architecture + +### Recommended Options + +#### 1. **nginx** (Recommended for Kubernetes) +- **Pros**: Excellent gRPC support, lightweight, Kubernetes Ingress controller +- **Cons**: More complex configuration for advanced routing +- **Use Case**: Kubernetes deployments, high-performance edge routing + +#### 2. **HAProxy** (Recommended for VM/Bare Metal) +- **Pros**: Battle-tested, advanced health checks, transparent gRPC support +- **Cons**: Steeper learning curve +- **Use Case**: VM-based deployments, multi-datacenter setups + +#### 3. **AWS Application Load Balancer (ALB)** +- **Pros**: Managed service, auto-scaling, AWS integration +- **Cons**: Vendor lock-in, limited gRPC support (requires workarounds) +- **Use Case**: AWS-hosted deployments + +#### 4. **Envoy Proxy** (Advanced) +- **Pros**: Native gRPC support, service mesh ready, advanced observability +- **Cons**: Complex configuration, resource overhead +- **Use Case**: Microservices mesh, multi-protocol routing + +### gRPC Load Balancing Requirements + +**Critical**: gRPC uses HTTP/2 with persistent connections, requiring **Layer 7 (L7) load balancing**. + +**Why L4 Fails**: +``` +Client ─── [L4 LB] ───> Backend 1 (gets ALL traffic) + β”‚ Backend 2 (idle) + β”‚ Backend 3 (idle) + └─ Single TCP connection = Single backend +``` + +**Why L7 Works**: +``` +Client ─── [L7 LB] ───> Backend 1 (33% requests) + β”‚ Backend 2 (33% requests) + β”‚ Backend 3 (34% requests) + └─ Request-level balancing across HTTP/2 stream +``` + +### Health Check Configuration + +**Health Check Requirements**: +1. **Protocol**: gRPC Health Checking Protocol (standard) +2. **Interval**: Every 5 seconds +3. **Timeout**: 2 seconds +4. **Threshold**: 2 consecutive failures = unhealthy +5. **Path**: `grpc.health.v1.Health/Check` + +**Example Health Check**: +```bash +# Using grpc_health_probe +grpc_health_probe -addr=api-gateway:50051 + +# Response +status: SERVING +``` + +### Session Affinity (Sticky Sessions) + +**When to Use**: +- Stateful operations (rare in our architecture) +- WebSocket connections (if added later) +- User-specific caching + +**When NOT to Use**: +- Stateless services (Trading, Backtesting, ML Training) +- High-availability scenarios (sticky sessions reduce failover capability) + +**Configuration**: +```nginx +# nginx session affinity (if needed) +upstream api_gateway { + ip_hash; # Session affinity based on client IP + server gateway-1:50051; + server gateway-2:50051; +} +``` + +--- + +## API Gateway Load Balancing + +### Single Entry Point Pattern + +**Architecture**: +``` +External Clients (TLI, Mobile, Web) + | + v + Load Balancer (TLS termination) + | + v + API Gateway Cluster (N instances) + | + v + Backend Services (Trading, Backtesting, ML) +``` + +### TLS Termination at Load Balancer + +**Benefits**: +1. **Reduced Backend Load**: API Gateway doesn't handle TLS handshakes +2. **Certificate Management**: Single point for certificate renewal +3. **Simplified Backend**: Backend services can use plaintext gRPC +4. **Performance**: Hardware-accelerated TLS at load balancer + +**Configuration**: +```nginx +# nginx TLS termination +server { + listen 443 ssl http2; + + ssl_certificate /etc/ssl/certs/foxhunt.crt; + ssl_certificate_key /etc/ssl/private/foxhunt.key; + + ssl_protocols TLSv1.3 TLSv1.2; + ssl_ciphers HIGH:!aNULL:!MD5; + + location / { + grpc_pass grpc://api_gateway_backend; + } +} +``` + +### Rate Limiting at Edge + +**Why Edge Rate Limiting**: +1. **DDoS Protection**: Block attack traffic before it reaches backend +2. **Resource Efficiency**: Save backend resources for legitimate requests +3. **Fairness**: Prevent single client from monopolizing resources + +**Rate Limit Tiers**: +```nginx +# nginx rate limiting by tier +limit_req_zone $jwt_user_tier zone=tier1:10m rate=100r/s; # Free tier +limit_req_zone $jwt_user_tier zone=tier2:10m rate=500r/s; # Premium +limit_req_zone $jwt_user_tier zone=tier3:10m rate=2000r/s; # Enterprise + +location /api { + limit_req zone=tier1 burst=20 nodelay; + grpc_pass grpc://api_gateway_backend; +} +``` + +**Rate Limit Configuration**: +| Tier | Rate Limit | Burst | Typical Use Case | +|------------|------------|-------|------------------| +| Free | 100 req/s | 20 | Retail traders | +| Premium | 500 req/s | 100 | Active traders | +| Enterprise | 2000 req/s | 500 | HFT firms | +| Internal | Unlimited | N/A | Service-to-service | + +### DDoS Protection + +**Layer 3/4 Protection** (Network/Transport): +```nginx +# nginx connection limits +limit_conn_zone $binary_remote_addr zone=addr:10m; +limit_conn addr 10; # Max 10 concurrent connections per IP +``` + +**Layer 7 Protection** (Application): +```nginx +# Rate limiting (see above) +# Request size limits +client_max_body_size 1M; + +# Timeout limits +grpc_read_timeout 30s; +grpc_send_timeout 30s; +``` + +**Additional Measures**: +1. **IP Blacklisting**: Block known malicious IPs +2. **Geo-Blocking**: Restrict access by country (if needed) +3. **Challenge-Response**: CAPTCHA for suspicious traffic +4. **Cloud-Based DDoS Protection**: Cloudflare, AWS Shield + +--- + +## Backend Service Scaling + +### Trading Service: Stateless Scaling + +**Characteristics**: +- **Stateless**: No in-memory state (all state in PostgreSQL/Redis) +- **Horizontal Scaling**: Add instances linearly for more capacity +- **Auto-Scaling Trigger**: CPU > 70%, Request Rate > 1000 req/s + +**Scaling Strategy**: +```yaml +# Kubernetes HPA for Trading Service +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: trading-service-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: trading-service + minReplicas: 3 # Always maintain 3 for HA + maxReplicas: 20 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + - type: Pods + pods: + metric: + name: grpc_requests_per_second + target: + type: AverageValue + averageValue: "1000" +``` + +**Resource Requirements**: +```yaml +resources: + requests: + cpu: 500m # 0.5 CPU core + memory: 1Gi # 1GB RAM + limits: + cpu: 2000m # 2 CPU cores + memory: 4Gi # 4GB RAM +``` + +**Expected Performance Per Instance**: +- **Throughput**: 1,000-2,000 requests/second +- **Latency**: P99 < 5ms (order submission) +- **Concurrent Connections**: 1,000-5,000 + +### Backtesting Service: Job Queue Pattern + +**Characteristics**: +- **Long-Running Jobs**: Backtests can take minutes to hours +- **CPU-Intensive**: High CPU usage during simulation +- **Job Queue**: Use Redis queue for job distribution + +**Scaling Strategy**: +```yaml +# Kubernetes HPA for Backtesting Service (worker pool) +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: backtesting-service-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: backtesting-service + minReplicas: 2 # Minimum workers + maxReplicas: 10 # Maximum workers + metrics: + - type: Pods + pods: + metric: + name: redis_queue_length # Scale on queue depth + target: + type: AverageValue + averageValue: "5" # 5 jobs per worker +``` + +**Job Queue Design**: +``` +Client Request β†’ API Gateway β†’ Backtesting Service (Job Scheduler) + ↓ + Redis Job Queue + ↓ + Worker Pool (N instances) + ↓ ↓ ↓ + Worker 1 Worker 2 Worker N + ↓ ↓ ↓ + Results stored in PostgreSQL +``` + +**Resource Requirements** (per worker): +```yaml +resources: + requests: + cpu: 2000m # 2 CPU cores (CPU-intensive) + memory: 4Gi # 4GB RAM (large datasets) + limits: + cpu: 4000m # 4 CPU cores + memory: 8Gi # 8GB RAM +``` + +### ML Training Service: GPU Resource Allocation + +**Characteristics**: +- **GPU-Dependent**: Requires NVIDIA GPU (RTX 3050 Ti or better) +- **Long-Running Jobs**: Training can take hours to days +- **Resource-Intensive**: High GPU memory, CPU, and RAM usage + +**Scaling Strategy** (Kubernetes with NVIDIA GPU Operator): +```yaml +# ML Training Service with GPU affinity +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ml-training-service +spec: + replicas: 2 # Manual scaling (GPU resources expensive) + template: + spec: + containers: + - name: ml-training-service + image: foxhunt/ml-training-service:latest + resources: + requests: + cpu: 4000m # 4 CPU cores + memory: 16Gi # 16GB RAM + nvidia.com/gpu: 1 # 1 GPU per pod + limits: + cpu: 8000m # 8 CPU cores + memory: 32Gi # 32GB RAM + nvidia.com/gpu: 1 + env: + - name: CUDA_VISIBLE_DEVICES + value: "0" +``` + +**GPU Scheduling Considerations**: +1. **Node Affinity**: Schedule only on GPU-enabled nodes +2. **GPU Sharing**: Use NVIDIA MPS for multi-tenant GPU sharing (if needed) +3. **Preemption**: Low-priority training jobs can be preempted for inference +4. **Cost**: GPU instances are 5-10x more expensive than CPU + +**Auto-Scaling NOT Recommended**: +- GPU instances take 2-5 minutes to provision +- Training jobs are scheduled, not real-time +- Use job queue pattern (like Backtesting Service) + +### Database Connection Pooling + +**Why Connection Pooling**: +- PostgreSQL connection limit: ~100-500 connections +- Each service instance needs 5-20 connections +- 20 service instances Γ— 20 connections = 400 connections (nearing limit) + +**Solution: PgBouncer**: +``` +Service Instances (100+) β†’ PgBouncer (pools to 50 connections) β†’ PostgreSQL +``` + +**PgBouncer Configuration**: +```ini +# pgbouncer.ini +[databases] +foxhunt = host=postgres port=5432 dbname=foxhunt + +[pgbouncer] +listen_addr = 0.0.0.0 +listen_port = 6432 +auth_type = scram-sha-256 +auth_file = /etc/pgbouncer/userlist.txt + +# Connection pooling +pool_mode = transaction # Transaction-level pooling +max_client_conn = 1000 # Max connections from services +default_pool_size = 25 # Pool size per database +reserve_pool_size = 10 # Emergency pool +``` + +**Service Configuration** (use PgBouncer instead of direct PostgreSQL): +```bash +# Before (direct PostgreSQL) +DATABASE_URL=postgresql://foxhunt:password@postgres:5432/foxhunt + +# After (via PgBouncer) +DATABASE_URL=postgresql://foxhunt:password@pgbouncer:6432/foxhunt +``` + +--- + +## Horizontal Pod Autoscaling (HPA) + +### Metrics for Scaling Decisions + +**CPU-Based Scaling** (Most Common): +```yaml +metrics: +- type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 # Scale when CPU > 70% +``` + +**Memory-Based Scaling**: +```yaml +metrics: +- type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 # Scale when memory > 80% +``` + +**Custom Metrics** (Request Rate): +```yaml +metrics: +- type: Pods + pods: + metric: + name: grpc_requests_per_second + target: + type: AverageValue + averageValue: "1000" # Scale when > 1000 req/s per pod +``` + +**Custom Metrics** (Queue Depth): +```yaml +metrics: +- type: Pods + pods: + metric: + name: redis_queue_length + target: + type: AverageValue + averageValue: "5" # Scale when > 5 jobs per worker +``` + +### Scale-Up Thresholds + +**Aggressive Scale-Up** (Trading Service - latency-sensitive): +```yaml +behavior: + scaleUp: + stabilizationWindowSeconds: 30 # Wait 30s before scaling up + policies: + - type: Percent + value: 100 # Double replicas + periodSeconds: 15 + - type: Pods + value: 4 # Or add 4 pods + periodSeconds: 15 + selectPolicy: Max # Use whichever scales faster +``` + +**Conservative Scale-Up** (Backtesting Service - cost-sensitive): +```yaml +behavior: + scaleUp: + stabilizationWindowSeconds: 60 # Wait 60s before scaling up + policies: + - type: Percent + value: 50 # Add 50% more replicas + periodSeconds: 60 + - type: Pods + value: 2 # Or add 2 pods + periodSeconds: 60 + selectPolicy: Min # Use whichever scales slower +``` + +### Scale-Down Thresholds + +**Slow Scale-Down** (Prevent flapping): +```yaml +behavior: + scaleDown: + stabilizationWindowSeconds: 300 # Wait 5 minutes before scaling down + policies: + - type: Percent + value: 25 # Remove 25% of replicas + periodSeconds: 60 + - type: Pods + value: 1 # Or remove 1 pod + periodSeconds: 60 + selectPolicy: Min # Use whichever scales slower +``` + +**Why Slow Scale-Down**: +1. **Prevent Flapping**: Avoid rapid up/down cycles (costs CPU, disrupts connections) +2. **Traffic Spikes**: Allow system to handle sudden load increases +3. **Cost**: Over-provisioning by 10-20% is cheaper than SLA violations + +### Min/Max Replica Counts + +**Recommended Configurations**: + +| Service | Min Replicas | Max Replicas | Reasoning | +|-------------------|--------------|--------------|-----------| +| API Gateway | 3 | 20 | HA (3) + burst capacity | +| Trading Service | 3 | 20 | HA (3) + 50K orders/sec | +| Backtesting | 2 | 10 | Workers for job queue | +| ML Training | 1 | 3 | GPU resources expensive | + +**Min Replicas Rationale**: +- **3 for HA**: Survive 1 node failure + 1 rolling update +- **2 for Workers**: Minimum for job queue parallelism +- **1 for GPU**: GPU resources too expensive for idle HA + +**Max Replicas Rationale**: +- **20 for Trading**: Based on performance testing (50K orders/sec Γ· 2.5K orders/sec per pod) +- **10 for Backtesting**: Rarely need more than 10 concurrent backtests +- **3 for ML Training**: GPU cluster size limit + +--- + +## Database Scaling + +### Read Replicas for Read-Heavy Workloads + +**When to Use Read Replicas**: +- **Read:Write Ratio**: > 3:1 (75%+ reads) +- **Query Patterns**: Analytics, reporting, backtesting +- **Write Latency**: Not affected by read load + +**PostgreSQL Streaming Replication**: +``` +Primary (Write) ──────> Replica 1 (Read) + β”‚ Replica 2 (Read) + β”‚ Replica 3 (Read) + └─ Async replication (< 100ms lag) +``` + +**Configuration**: +```yaml +# Kubernetes StatefulSet with read replicas +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres-replica +spec: + replicas: 2 # 2 read replicas + serviceName: postgres-replica + template: + spec: + containers: + - name: postgres + image: timescale/timescaledb:latest-pg15 + env: + - name: POSTGRES_REPLICA_MODE + value: "replica" + - name: POSTGRES_MASTER_HOST + value: "postgres-primary" +``` + +**Application-Level Read/Write Splitting**: +```rust +// trading_engine/src/persistence/postgres.rs +pub struct PostgresPool { + write_pool: sqlx::PgPool, // Primary + read_pool: sqlx::PgPool, // Read replica(s) +} + +impl PostgresPool { + pub async fn execute_write(&self, query: Query) -> Result<()> { + // All writes go to primary + self.write_pool.execute(query).await + } + + pub async fn execute_read(&self, query: Query) -> Result { + // Reads go to replica (or primary if replica unavailable) + self.read_pool.fetch_all(query).await + } +} +``` + +### Connection Pooling (PgBouncer) + +**See "Backend Service Scaling" section for PgBouncer configuration.** + +**Benefits**: +1. **Reduced Connection Overhead**: Reuse connections instead of creating new ones +2. **Connection Limit Management**: Protect PostgreSQL from connection exhaustion +3. **Transaction Pooling**: Release connections immediately after transaction + +**Pooling Modes**: +```ini +# Transaction pooling (recommended for stateless services) +pool_mode = transaction # Connection released after transaction + +# Session pooling (for stateful operations) +pool_mode = session # Connection held for entire session + +# Statement pooling (most aggressive, use with caution) +pool_mode = statement # Connection released after each statement +``` + +### Sharding Strategy (Future - If Needed) + +**When to Shard**: +- Database size > 1TB +- Write throughput > 10K writes/sec +- Read replicas insufficient + +**Sharding Strategies**: + +1. **By Asset Class** (Natural partition): +```sql +-- Shard 1: Equities +-- Shard 2: Options +-- Shard 3: Futures +-- Shard 4: Crypto +``` + +2. **By Time** (TimescaleDB hypertables - automatic): +```sql +-- TimescaleDB automatically partitions by time +CREATE TABLE market_data ( + time TIMESTAMPTZ NOT NULL, + symbol TEXT NOT NULL, + price DOUBLE PRECISION +); + +SELECT create_hypertable('market_data', 'time'); +-- Automatic partitioning by month/week +``` + +3. **By User ID** (Hash-based): +```sql +-- Shard based on user_id hash +Shard = user_id % num_shards +``` + +**Recommendation**: Start with TimescaleDB hypertables (time-based partitioning). Only implement custom sharding if necessary. + +### TimescaleDB Hypertables + +**Already Configured** (from migrations): +```sql +-- Migration 002: TimescaleDB extension +CREATE EXTENSION IF NOT EXISTS timescaledb; + +-- Migration 003: Market data hypertable +SELECT create_hypertable('market_data', 'timestamp', + chunk_time_interval => INTERVAL '1 day'); + +-- Automatic compression for old data +ALTER TABLE market_data SET ( + timescaledb.compress, + timescaledb.compress_orderby = 'timestamp DESC', + timescaledb.compress_segmentby = 'symbol' +); +``` + +**Benefits**: +- **Automatic Partitioning**: By time (day/week/month) +- **Compression**: Old data automatically compressed (10x space savings) +- **Parallel Queries**: TimescaleDB parallelizes across chunks +- **Retention Policies**: Automatic data deletion after N days + +--- + +## Redis Scaling + +### Redis Cluster for Horizontal Scaling + +**When to Use Redis Cluster**: +- Dataset size > 10GB (single node RAM limit) +- Throughput > 100K ops/sec +- High availability required + +**Redis Cluster Architecture**: +``` +Client ──> Redis Cluster Proxy + β”‚ + β”œβ”€β”€> Shard 1 (Master + Replica) + β”œβ”€β”€> Shard 2 (Master + Replica) + └──> Shard 3 (Master + Replica) +``` + +**Configuration** (docker-compose.yml): +```yaml +services: + redis-cluster: + image: redis:7-alpine + command: redis-server --cluster-enabled yes --cluster-config-file nodes.conf + ports: + - "7000-7005:7000-7005" + volumes: + - redis-cluster-data:/data + + # Create cluster (run once) + redis-cluster-init: + image: redis:7-alpine + command: | + redis-cli --cluster create \ + redis-1:7000 redis-2:7001 redis-3:7002 \ + redis-4:7003 redis-5:7004 redis-6:7005 \ + --cluster-replicas 1 + depends_on: + - redis-cluster +``` + +**Client Configuration**: +```rust +// Use redis-cluster-async crate +use redis_cluster_async::{Client, ClusterClient}; + +let client = ClusterClient::new(vec![ + "redis://redis-1:7000", + "redis://redis-2:7001", + "redis://redis-3:7002", +])?; + +// Client automatically routes to correct shard +let mut conn = client.get_connection().await?; +conn.set("key", "value").await?; +``` + +### Redis Sentinel for High Availability + +**When to Use Sentinel** (Alternative to Cluster): +- Single master + replicas (no sharding) +- Automatic failover required +- Dataset fits in single node RAM + +**Sentinel Architecture**: +``` +Client ──> Sentinel (monitors health) + β”‚ + v + Primary ──> Replica 1 + └─> Replica 2 + +(If Primary fails, Sentinel promotes Replica 1) +``` + +**Configuration**: +```yaml +# docker-compose.yml +services: + redis-primary: + image: redis:7-alpine + ports: + - "6379:6379" + + redis-replica-1: + image: redis:7-alpine + command: redis-server --replicaof redis-primary 6379 + + redis-sentinel: + image: redis:7-alpine + command: redis-sentinel /etc/redis/sentinel.conf + volumes: + - ./redis-sentinel.conf:/etc/redis/sentinel.conf +``` + +```conf +# redis-sentinel.conf +sentinel monitor mymaster redis-primary 6379 2 +sentinel down-after-milliseconds mymaster 5000 +sentinel parallel-syncs mymaster 1 +sentinel failover-timeout mymaster 10000 +``` + +### Cache Warming Strategies + +**Cold Cache Problem**: +- Cache miss rate spikes after restart +- Database overload from cache misses +- Slow response times until cache warms + +**Solution 1: Lazy Loading** (Default): +```rust +// Check cache first, load from DB on miss +async fn get_user_config(user_id: &str, cache: &RedisPool, db: &PgPool) -> Result { + // Try cache first + if let Some(config) = cache.get(user_id).await? { + return Ok(config); + } + + // Cache miss: Load from database + let config = db.fetch_one("SELECT * FROM user_config WHERE user_id = $1", user_id).await?; + + // Warm cache for next request + cache.set(user_id, &config, Duration::from_secs(3600)).await?; + + Ok(config) +} +``` + +**Solution 2: Proactive Warming**: +```rust +// Warm cache on startup with most-used data +async fn warm_cache(cache: &RedisPool, db: &PgPool) -> Result<()> { + // Load top 1000 most active users + let active_users = db.fetch_all( + "SELECT user_id, config FROM user_config ORDER BY last_active DESC LIMIT 1000" + ).await?; + + for user in active_users { + cache.set(&user.user_id, &user.config, Duration::from_secs(3600)).await?; + } + + Ok(()) +} +``` + +**Solution 3: Cache Priming** (Scheduled): +```rust +// Periodically refresh cache for hot data +async fn prime_cache_job(cache: &RedisPool, db: &PgPool) -> Result<()> { + loop { + // Refresh market data cache every minute + let latest_quotes = db.fetch_all("SELECT * FROM latest_quotes").await?; + for quote in latest_quotes { + cache.set("e.symbol, "e, Duration::from_secs(60)).await?; + } + + tokio::time::sleep(Duration::from_secs(60)).await; + } +} +``` + +### Eviction Policies + +**Redis Eviction Policies**: +```conf +# redis.conf +maxmemory 2gb +maxmemory-policy allkeys-lru # Evict least recently used keys +``` + +**Policy Options**: +| Policy | Description | Use Case | +|-------------------|-------------|----------| +| `noeviction` | Return error when memory full | Never (avoid data loss) | +| `allkeys-lru` | Evict least recently used keys | **Recommended for cache** | +| `allkeys-lfu` | Evict least frequently used keys | Hot key workloads | +| `volatile-lru` | Evict LRU keys with TTL set | Mixed cache + persistent data | +| `volatile-ttl` | Evict keys with shortest TTL | Time-sensitive data | + +**Recommendation**: Use `allkeys-lru` for cache workloads. + +--- + +## Performance Targets + +### Latency Targets (from PERFORMANCE_BENCHMARKS.md) + +| Operation | P50 | P99 | P99.9 | +|-----------|-----|-----|-------| +| Order Submission | < 1ms | < 5ms | < 10ms | +| Market Data Ingestion | < 500ΞΌs | < 2ms | < 5ms | +| Position Query | < 2ms | < 10ms | < 20ms | +| Risk Check | < 1ms | < 5ms | < 10ms | + +### Throughput Targets + +| Service | Target Throughput | Peak Capacity | +|---------|------------------|---------------| +| API Gateway | 10K req/s | 50K req/s | +| Trading Service | 5K orders/s | 20K orders/s | +| Market Data Ingestion | 100K events/s | 500K events/s | +| Backtesting Service | 10 concurrent backtests | 50 concurrent backtests | + +**Scaling Math**: +``` +# Trading Service Example +Target: 20K orders/sec +Per-instance capacity: 2K orders/sec +Required instances: 20K Γ· 2K = 10 instances +Safety margin (20%): 10 Γ— 1.2 = 12 instances +``` + +### Concurrent User Limits + +| User Tier | Concurrent Connections | Rate Limit | +|-----------|----------------------|------------| +| Free | 1 | 100 req/s | +| Premium | 5 | 500 req/s | +| Enterprise | 50 | 2000 req/s | + +### Resource Utilization Targets + +**Target Utilization** (for auto-scaling): +- **CPU**: 60-70% (allows headroom for bursts) +- **Memory**: 70-80% (allows headroom for spikes) +- **Disk**: < 80% (alerts at 80%, critical at 90%) +- **Network**: < 50% (network saturation kills performance) + +**Why NOT 90% Utilization**: +1. **No Headroom**: No capacity for traffic spikes +2. **Slow Auto-Scaling**: Takes 30-60s to add instances +3. **Latency Degradation**: High CPU = increased latency +4. **OOM Kills**: High memory usage risks out-of-memory crashes + +--- + +## Cost Optimization + +### Right-Sizing Instances + +**CPU/Memory Ratios by Service**: + +| Service | CPU | Memory | Ratio | Instance Type (AWS) | +|---------|-----|--------|-------|---------------------| +| API Gateway | 1 core | 2GB | 1:2 | t3.small | +| Trading Service | 2 cores | 4GB | 1:2 | t3.medium | +| Backtesting Service | 4 cores | 8GB | 1:2 | c6i.xlarge (compute-optimized) | +| ML Training Service | 8 cores | 32GB + GPU | 1:4 | g4dn.2xlarge (GPU) | + +**How to Right-Size**: +1. **Monitor Actual Usage**: Use Prometheus metrics for 2 weeks +2. **Identify Bottlenecks**: CPU-bound vs memory-bound vs I/O-bound +3. **Adjust Instance Types**: Match instance type to workload +4. **Iterate**: Re-evaluate every quarter + +### Spot Instances for Non-Critical Workloads + +**What are Spot Instances**: +- Spare cloud capacity sold at 50-90% discount +- Can be terminated with 2-minute notice +- **Only use for fault-tolerant workloads** + +**Recommended Use Cases**: +```yaml +# Kubernetes node groups +nodeGroups: + # Production workloads (on-demand) + - name: production-on-demand + instanceType: t3.medium + desiredCapacity: 10 + labels: + workload: production + + # Backtesting workers (spot instances) + - name: backtesting-spot + instanceType: c6i.xlarge + desiredCapacity: 5 + instancesDistribution: + onDemandBaseCapacity: 1 # Always keep 1 on-demand + onDemandPercentageAboveBaseCapacity: 0 # Rest are spot + spotAllocationStrategy: capacity-optimized + labels: + workload: backtesting +``` + +**Never Use Spot For**: +- Trading Service (real-time, SLA-critical) +- API Gateway (user-facing) +- Databases (stateful) + +**Good Use Cases**: +- Backtesting workers (jobs can retry) +- ML training (checkpointing allows resume) +- Batch processing + +### Reserved Instances for Base Load + +**What are Reserved Instances**: +- 1-year or 3-year commitment +- 30-60% discount vs on-demand +- Best for predictable, steady workload + +**Recommended Strategy**: +``` +Total Capacity Needed: 20 instances +β”œβ”€β”€ Base Load (always needed): 10 instances β†’ Reserved (50% savings) +β”œβ”€β”€ Typical Load: +5 instances β†’ On-Demand +└── Peak Load: +5 instances β†’ Auto-Scaling (On-Demand or Spot) +``` + +**Cost Comparison** (AWS t3.medium, us-east-1): +| Pricing Model | Cost/month | Savings | +|---------------|------------|---------| +| On-Demand | $30.37 | 0% | +| Reserved (1yr, no upfront) | $18.25 | 40% | +| Reserved (3yr, all upfront) | $11.68 | 62% | +| Spot (average) | $9.11 | 70% | + +### Auto-Scaling for Burst Capacity + +**Cost-Optimized Auto-Scaling**: +```yaml +# HPA with cost awareness +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: trading-service-hpa +spec: + minReplicas: 10 # Reserved instances (base load) + maxReplicas: 20 # Auto-scale to 20 (on-demand/spot) + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + + # Slow scale-down to avoid flapping + behavior: + scaleDown: + stabilizationWindowSeconds: 600 # 10 minutes + policies: + - type: Pods + value: 1 + periodSeconds: 120 # Remove 1 pod every 2 minutes +``` + +**Cost Math**: +``` +Scenario 1: No Auto-Scaling (always 20 instances) +Cost: 20 Γ— $30.37 = $607.40/month + +Scenario 2: Auto-Scaling (10 reserved, 10 auto-scale) +Base: 10 Γ— $18.25 = $182.50 (reserved) +Burst: 10 Γ— $30.37 Γ— 30% = $91.11 (on-demand, 30% of time) +Total: $273.61/month +Savings: $333.79/month (55%) +``` + +--- + +## Monitoring & Alerting + +### Key Metrics to Monitor + +**Service Health**: +```promql +# Service availability (should be > 99.9%) +up{job="api-gateway"} == 1 + +# Request rate +rate(grpc_requests_total[5m]) + +# Error rate (should be < 0.1%) +rate(grpc_requests_total{status="error"}[5m]) / rate(grpc_requests_total[5m]) + +# Latency (P99 should be < 5ms) +histogram_quantile(0.99, rate(grpc_request_duration_seconds_bucket[5m])) +``` + +**Auto-Scaling Metrics**: +```promql +# Current replica count +kube_deployment_status_replicas{deployment="trading-service"} + +# Desired replica count (from HPA) +kube_horizontalpodautoscaler_status_desired_replicas{hpa="trading-service-hpa"} + +# CPU utilization +avg(rate(container_cpu_usage_seconds_total{pod=~"trading-service.*"}[5m])) * 100 + +# Memory utilization +avg(container_memory_working_set_bytes{pod=~"trading-service.*"}) / +avg(container_spec_memory_limit_bytes{pod=~"trading-service.*"}) * 100 +``` + +**Database Metrics**: +```promql +# Connection pool usage (should be < 80%) +pg_stat_database_numbackends / pg_settings_max_connections * 100 + +# Replication lag (should be < 100ms) +pg_stat_replication_replay_lag_bytes + +# Query latency +rate(pg_stat_database_blks_read[5m]) +``` + +### Alerts to Configure + +**Critical Alerts** (PagerDuty): +```yaml +# Service down +- alert: ServiceDown + expr: up{job=~".*service"} == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Service {{ $labels.job }} is down" + +# High error rate +- alert: HighErrorRate + expr: rate(grpc_requests_total{status="error"}[5m]) / rate(grpc_requests_total[5m]) > 0.05 + for: 5m + labels: + severity: critical + annotations: + summary: "High error rate on {{ $labels.service }}: {{ $value | humanizePercentage }}" + +# High latency +- alert: HighLatency + expr: histogram_quantile(0.99, rate(grpc_request_duration_seconds_bucket[5m])) > 0.1 + for: 5m + labels: + severity: critical + annotations: + summary: "High P99 latency on {{ $labels.service }}: {{ $value }}s" +``` + +**Warning Alerts** (Slack): +```yaml +# High CPU usage (approaching auto-scale threshold) +- alert: HighCPU + expr: avg(rate(container_cpu_usage_seconds_total[5m])) by (pod) > 0.8 + for: 10m + labels: + severity: warning + annotations: + summary: "High CPU on {{ $labels.pod }}: {{ $value | humanizePercentage }}" + +# Database connection pool approaching limit +- alert: HighDatabaseConnections + expr: pg_stat_database_numbackends / pg_settings_max_connections > 0.8 + for: 5m + labels: + severity: warning + annotations: + summary: "Database connection pool at {{ $value | humanizePercentage }}" +``` + +### Dashboards + +**Recommended Grafana Dashboards**: +1. **Service Overview**: Request rate, error rate, latency (RED metrics) +2. **Auto-Scaling**: Current/desired replicas, CPU/memory utilization +3. **Database**: Connection pool, query latency, replication lag +4. **Load Balancer**: Active connections, requests per backend +5. **Cost**: Instance count, spot vs on-demand usage + +--- + +## Quick Reference + +### Scaling Decision Matrix + +| Symptom | Root Cause | Solution | +|---------|-----------|----------| +| High CPU (> 80%) | Not enough compute | Scale up replicas | +| High memory (> 90%) | Memory leak or under-provisioned | Increase memory limits or fix leak | +| High latency + low CPU | Database bottleneck | Add read replicas or optimize queries | +| High error rate | Service overload | Scale up replicas or rate limit clients | +| Database connection exhaustion | Too many service instances | Add PgBouncer connection pooling | +| Redis memory full | Cache size exceeds capacity | Add Redis Cluster or increase eviction | + +### Scaling Commands + +**Kubernetes**: +```bash +# Manual scaling +kubectl scale deployment trading-service --replicas=10 + +# Check HPA status +kubectl get hpa + +# Describe HPA (shows scaling events) +kubectl describe hpa trading-service-hpa + +# View pod resource usage +kubectl top pods + +# View node resource usage +kubectl top nodes +``` + +**Docker Compose** (development): +```bash +# Scale service +docker-compose up -d --scale trading_service=5 + +# View resource usage +docker stats +``` + +### Load Balancer Health Checks + +**nginx**: +```bash +# Check nginx status +curl http://localhost/nginx_status + +# Test gRPC endpoint +grpc_health_probe -addr=localhost:443 -tls +``` + +**HAProxy**: +```bash +# Check HAProxy stats +curl http://localhost:8404/stats + +# Test backend health +echo "show servers state" | socat /var/run/haproxy.sock stdio +``` + +--- + +## Additional Resources + +- **Kubernetes HPA**: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/ +- **nginx gRPC Load Balancing**: https://www.nginx.com/blog/nginx-1-13-10-grpc/ +- **PostgreSQL Connection Pooling**: https://www.pgbouncer.org/ +- **Redis Cluster**: https://redis.io/docs/manual/scaling/ +- **Performance Benchmarks**: See `PERFORMANCE_BENCHMARKS.md` +- **Scaling Playbook**: See `SCALING_PLAYBOOK.md` + +--- + +**Last Updated**: 2025-10-07 +**Version**: 1.0 +**Owner**: Platform Engineering Team diff --git a/PRODUCTION_DEPLOYMENT_RUNBOOK.md b/PRODUCTION_DEPLOYMENT_RUNBOOK.md new file mode 100644 index 000000000..64d139764 --- /dev/null +++ b/PRODUCTION_DEPLOYMENT_RUNBOOK.md @@ -0,0 +1,2311 @@ +# Production Deployment Runbook - Foxhunt HFT Trading System + +**Version**: 1.0.0 +**Last Updated**: 2025-10-07 +**Status**: Production Ready (93-94% deployment readiness) +**Owner**: DevOps Team + +--- + +## 🎯 Document Purpose + +This runbook provides comprehensive step-by-step procedures for deploying, operating, and maintaining the Foxhunt HFT Trading System in production environments. It covers all operational scenarios from initial deployment through disaster recovery. + +**Critical Reference**: This document incorporates findings from Agent 96's Docker E2E validation and addresses all identified deployment blockers. + +--- + +## πŸ“‹ Table of Contents + +1. [Prerequisites](#prerequisites) +2. [Initial Deployment](#initial-deployment) +3. [Rolling Updates](#rolling-updates) +4. [Scaling Procedures](#scaling-procedures) +5. [Disaster Recovery](#disaster-recovery) +6. [Monitoring and Alerting](#monitoring-and-alerting) +7. [Troubleshooting](#troubleshooting) +8. [Security Procedures](#security-procedures) + +--- + +## Prerequisites + +### 1. Infrastructure Requirements + +#### Hardware (Bare-Metal Deployment - Recommended for HFT) +- **CPU**: 16+ cores (Intel Xeon or AMD EPYC, 3.0+ GHz) +- **RAM**: 64GB+ minimum, 128GB recommended +- **Storage**: NVMe SSD 1TB+ (IOPS: 100K+ read, 50K+ write) +- **Network**: 10Gbps+ low-latency network adapter +- **GPU**: NVIDIA RTX 3050 Ti or better (for ML inference) + - CUDA 12.3+ support + - 4GB+ VRAM minimum + +#### Software Dependencies +```bash +# Operating System +Ubuntu 22.04 LTS (kernel 5.15+) + +# Container Runtime (if using Docker) +Docker Engine 24.0+ +Docker Compose 2.20+ +nvidia-docker2 (for GPU support) + +# Database +PostgreSQL 16+ with TimescaleDB extension +Redis 7.0+ + +# Monitoring +Prometheus 2.45+ +Grafana 10.0+ +InfluxDB 2.7+ (time-series metrics) + +# Secrets Management +HashiCorp Vault 1.15+ + +# Build Tools (for bare-metal) +Rust 1.89+ +CUDA Toolkit 12.3+ (for GPU services) +protobuf-compiler 3.0+ +``` + +### 2. Environment Variables + +**Agent 96 Finding**: Missing environment variables caused service failures. All variables below are REQUIRED. + +#### Core Infrastructure (All Services) +```bash +# Database Configuration +DATABASE_URL=postgresql://foxhunt:@:5432/foxhunt +REDIS_URL=redis://:6379 +VAULT_ADDR=http://:8200 +VAULT_TOKEN= + +# Logging +RUST_LOG=info +RUST_BACKTRACE=1 + +# Environment +FOXHUNT_ENV=production +``` + +#### Authentication Services (Trading, API Gateway) +```bash +# JWT Configuration +JWT_SECRET=<64+ character base64 string> +# Generate: openssl rand -base64 64 +# MUST contain: uppercase, lowercase, numbers, symbols + +# Production: Use file-based secrets instead +JWT_SECRET_FILE=/run/secrets/jwt_secret + +# JWT Claims +JWT_ISSUER=foxhunt-api-gateway +JWT_AUDIENCE=foxhunt-services +``` + +#### API Keys (Backtesting Service) +**Agent 96 Critical Finding**: Missing `BENZINGA_API_KEY` blocked backtesting service startup. + +```bash +# Required for Backtesting Service +BENZINGA_API_KEY= +# Obtain from: https://www.benzinga.com/apis + +# Optional: Databento API (already configured) +DATABENTO_API_KEY= +``` + +#### Security Tokens (Production) +```bash +# Kill Switch Authentication +KILL_SWITCH_MASTER_TOKEN= +# Generate: openssl rand -base64 32 + +# Production: Use file-based secrets +KILL_SWITCH_MASTER_TOKEN_FILE=/run/secrets/kill_switch_token +``` + +#### Rate Limiting (API Gateway) +```bash +RATE_LIMIT_RPS=100 +ENABLE_AUDIT_LOGGING=true +``` + +#### AWS S3 Configuration (Model Storage) +```bash +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_REGION=us-east-1 + +# S3 Buckets +S3_MODEL_STORAGE_BUCKET=foxhunt-models +S3_CHECKPOINT_BUCKET=foxhunt-checkpoints +S3_ARCHIVAL_BUCKET=foxhunt-archives +``` + +#### GPU Configuration (ML Services) +```bash +# CUDA Environment +CUDA_HOME=/usr/local/cuda +LD_LIBRARY_PATH=$CUDA_HOME/lib64:$CUDA_HOME/targets/x86_64-linux/lib +PATH=$CUDA_HOME/bin:$PATH + +# CUDA Build Flags (Docker) +CUDARC_CUDA_VERSION=13000 +CUDA_COMPUTE_CAP=86 # RTX 3050 Ti + +# Docker Runtime +NVIDIA_VISIBLE_DEVICES=all +NVIDIA_DRIVER_CAPABILITIES=compute,utility +``` + +### 3. Secrets Management (Vault Configuration) + +#### Initialize Vault Secrets +```bash +# Start Vault (development mode for initial setup) +docker run -d --name vault -p 8200:8200 \ + -e VAULT_DEV_ROOT_TOKEN_ID=foxhunt-dev-root \ + hashicorp/vault:1.15 server -dev + +# Set Vault address +export VAULT_ADDR=http://localhost:8200 +export VAULT_TOKEN=foxhunt-dev-root + +# Store JWT secret +vault kv put secret/foxhunt/jwt \ + secret="$(openssl rand -base64 64)" + +# Store kill switch token +vault kv put secret/foxhunt/kill_switch \ + master_token="$(openssl rand -base64 32)" + +# Store API keys +vault kv put secret/foxhunt/api_keys \ + benzinga="" \ + databento="" + +# Store AWS credentials +vault kv put secret/foxhunt/aws \ + access_key_id="" \ + secret_access_key="" + +# Verify secrets stored +vault kv list secret/foxhunt +``` + +#### Production Vault Configuration +```bash +# Production: Use proper Vault seal (not dev mode) +# See: https://www.vaultproject.io/docs/configuration/seal + +# Enable AppRole authentication +vault auth enable approle + +# Create policy for Foxhunt services +vault policy write foxhunt-services - < foxhunt +sudo chown -R foxhunt:foxhunt foxhunt +cd foxhunt + +# Build with CPU-specific optimizations +export RUSTFLAGS="-C target-cpu=native -C opt-level=3" +cargo build --release --workspace + +# Verify binaries +ls -lh target/release/ | grep -E "(trading|backtesting|ml_training|api_gateway)" + +# Expected output: +# trading_service ~50MB +# backtesting_service ~55MB +# ml_training_service ~80MB +# api_gateway ~45MB +``` + +#### Step 2: Install Binaries + +```bash +# Create installation directories +sudo mkdir -p /opt/foxhunt/{bin,config,logs,data,models,checkpoints} + +# Install binaries +sudo cp target/release/trading_service /opt/foxhunt/bin/ +sudo cp target/release/backtesting_service /opt/foxhunt/bin/ +sudo cp target/release/ml_training_service /opt/foxhunt/bin/ +sudo cp target/release/api_gateway /opt/foxhunt/bin/ + +# Set permissions +sudo chown -R foxhunt:foxhunt /opt/foxhunt +sudo chmod +x /opt/foxhunt/bin/* +``` + +#### Step 3: Configure SystemD Services + +**Agent 96 Note**: Services must start in correct order (infra β†’ backend β†’ gateway). + +```bash +# Generate SystemD service files +./deployment/create_systemd_services.sh + +# Service files created in /etc/systemd/system/: +# - foxhunt-trading.service +# - foxhunt-backtesting.service +# - foxhunt-ml-training.service +# - foxhunt-api-gateway.service + +# Reload SystemD +sudo systemctl daemon-reload + +# Enable services (auto-start on boot) +sudo systemctl enable foxhunt-trading +sudo systemctl enable foxhunt-backtesting +sudo systemctl enable foxhunt-ml-training +sudo systemctl enable foxhunt-api-gateway +``` + +#### Step 4: Database Setup + +```bash +# Start PostgreSQL (if not running) +sudo systemctl start postgresql +sudo systemctl enable postgresql + +# Create database and user +sudo -u postgres psql <'; +CREATE DATABASE foxhunt OWNER foxhunt; +\c foxhunt +CREATE EXTENSION IF NOT EXISTS timescaledb; +GRANT ALL PRIVILEGES ON DATABASE foxhunt TO foxhunt; +EOF + +# Run migrations +cd /opt/foxhunt +export DATABASE_URL=postgresql://foxhunt:@localhost:5432/foxhunt +cargo sqlx migrate run + +# Verify migrations +psql $DATABASE_URL -c '\dt' + +# Expected tables: +# - orders +# - positions +# - market_data +# - audit_logs +# - ml_models +# - configurations +# (+ 12 more from 18 migrations) +``` + +#### Step 5: Start Services (Correct Order) + +**Critical**: Services must start in dependency order to avoid panics. + +```bash +# 1. Infrastructure services +sudo systemctl start redis +sudo systemctl start vault +sudo systemctl status redis vault + +# 2. Backend services (parallel) +sudo systemctl start foxhunt-trading +sudo systemctl start foxhunt-backtesting +sudo systemctl start foxhunt-ml-training + +# Wait for backend health checks (30 seconds) +sleep 30 + +# 3. API Gateway (depends on all backends) +sudo systemctl start foxhunt-api-gateway + +# Verify all services running +sudo systemctl status foxhunt-* +``` + +#### Step 6: Health Validation + +```bash +# Run comprehensive health check +./deployment/health_check.sh --mode comprehensive --verbose + +# Expected output: +# βœ… Trading Service: HEALTHY (gRPC port 50052) +# βœ… Backtesting Service: HEALTHY (gRPC port 50053) +# βœ… ML Training Service: HEALTHY (gRPC port 50054) +# βœ… API Gateway: HEALTHY (gRPC port 50051) +# βœ… PostgreSQL: Connected +# βœ… Redis: Connected +# βœ… Vault: Sealed/Unsealed status +# βœ… Memory usage: <500MB per service +# βœ… CPU usage: <10% idle +``` + +#### Step 7: Smoke Tests + +```bash +# Test API Gateway +grpcurl -plaintext localhost:50051 grpc.health.v1.Health/Check + +# Expected: {"status": "SERVING"} + +# Test Trading Service (via Gateway) +# Requires TLI or gRPC client + +# Check metrics endpoints +curl http://localhost:9092/metrics | grep foxhunt_ +curl http://localhost:9093/metrics | grep foxhunt_ +curl http://localhost:9094/metrics | grep foxhunt_ +curl http://localhost:9091/metrics | grep foxhunt_ + +# Verify logs for errors +sudo journalctl -u foxhunt-* --since "5 minutes ago" | grep -i error + +# Should return: No errors +``` + +--- + +### Docker Deployment + +**Agent 96 Findings**: 3 critical Dockerfile issues must be fixed before deployment. + +#### Pre-Deployment Fixes (REQUIRED) + +**Fix 1: Dockerfile Path Corrections** +```bash +# Issue: Dockerfiles reference crates/config (doesn't exist) +# Fix: Update all Dockerfiles + +# Edit each Dockerfile: +# - services/trading_service/Dockerfile +# - services/backtesting_service/Dockerfile +# - services/ml_training_service/Dockerfile +# - services/api_gateway/Dockerfile + +# Find and replace: +# OLD: COPY crates/config ./crates/config +# NEW: COPY config ./config +``` + +**Fix 2: ML Training Service Entry Point** +```bash +# Issue: Container expects subcommand but doesn't provide default +# Fix: Add CMD to Dockerfile + +# Edit: services/ml_training_service/Dockerfile +# Add before final line: +CMD ["ml_training_service", "serve"] +``` + +**Fix 3: Add Benzinga API Key** +```bash +# Issue: Backtesting service requires BENZINGA_API_KEY +# Fix: Add to docker-compose.yml or .env file + +# Edit: docker-compose.yml +backtesting_service: + environment: + - BENZINGA_API_KEY=${BENZINGA_API_KEY} + +# Or add to .env: +echo "BENZINGA_API_KEY=" >> .env +``` + +#### Step 1: Build Docker Images + +```bash +cd /opt/foxhunt + +# Build all services +./deployment/build_docker_images.sh --service all --enable-gpu + +# Or build individually +docker build -f services/trading_service/Dockerfile -t foxhunt-trading-service:latest . +docker build -f services/backtesting_service/Dockerfile -t foxhunt-backtesting-service:latest . +docker build -f services/ml_training_service/Dockerfile -t foxhunt-ml-training-service:latest . +docker build -f services/api_gateway/Dockerfile -t foxhunt-api-gateway:latest . + +# Verify images built +docker images | grep foxhunt + +# Expected: +# foxhunt-trading-service latest 119MB +# foxhunt-backtesting-service latest 120MB +# foxhunt-ml-training-service latest 2.24GB (includes CUDA) +# foxhunt-api-gateway latest 119MB +``` + +#### Step 2: Configure Environment + +```bash +# Create production environment file +cp .env.example .env.production + +# Edit .env.production with production values +vim .env.production + +# Required variables (from Agent 96 findings): +DATABASE_URL=postgresql://foxhunt:@postgres:5432/foxhunt +REDIS_URL=redis://redis:6379 +VAULT_ADDR=http://vault:8200 +VAULT_TOKEN= +JWT_SECRET= +BENZINGA_API_KEY= +KILL_SWITCH_MASTER_TOKEN= +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= + +# Validate all required variables set +./deployment/scripts/validate-env.sh .env.production +``` + +#### Step 3: Start Infrastructure Services + +```bash +# Start database and caching layers +docker-compose up -d postgres redis vault influxdb + +# Wait for health checks +docker-compose ps + +# Verify all healthy (retry_count 0/5) +# foxhunt-postgres healthy +# foxhunt-redis healthy +# foxhunt-vault healthy +# foxhunt-influxdb healthy +``` + +#### Step 4: Run Database Migrations + +```bash +# Connect to running postgres container +docker-compose exec postgres psql -U foxhunt -d foxhunt + +# Or run migrations from host +export DATABASE_URL=postgresql://foxhunt:@localhost:5432/foxhunt +cargo sqlx migrate run + +# Verify 18 migrations applied +psql $DATABASE_URL -c "SELECT version FROM _sqlx_migrations ORDER BY version;" +``` + +#### Step 5: Start Monitoring Stack + +```bash +# Start Prometheus and Grafana +docker-compose up -d prometheus grafana + +# Verify access +curl http://localhost:9090/-/healthy # Prometheus +curl http://localhost:3000/api/health # Grafana + +# Login to Grafana +# URL: http://localhost:3000 +# Username: admin +# Password: foxhunt123 (from docker-compose.yml) +``` + +#### Step 6: Start Application Services (Correct Order) + +```bash +# Start backend services (can run in parallel) +docker-compose up -d trading_service backtesting_service ml_training_service + +# Wait for services to initialize (30-60 seconds for GPU services) +sleep 60 + +# Check logs for successful startup +docker-compose logs trading_service | grep "gRPC server started" +docker-compose logs backtesting_service | grep "Repository layer initialized" +docker-compose logs ml_training_service | grep "ML training service started" + +# Start API Gateway (after backends healthy) +docker-compose up -d api_gateway + +# Verify all services running +docker-compose ps + +# Expected: +# foxhunt-trading-service Up 0.0.0.0:50052->50051/tcp +# foxhunt-backtesting-service Up 0.0.0.0:50053->50052/tcp +# foxhunt-ml-training-service Up 0.0.0.0:50054->50053/tcp +# foxhunt-api-gateway Up 0.0.0.0:50051->50050/tcp +``` + +#### Step 7: Validate Deployment + +```bash +# Check service health +for port in 50051 50052 50053 50054; do + docker run --rm --network host \ + fullstorydev/grpcurl:latest \ + -plaintext localhost:$port grpc.health.v1.Health/Check +done + +# Expected: {"status": "SERVING"} for all + +# Check GPU access (ML Training Service) +docker-compose exec ml_training_service nvidia-smi + +# Expected: GPU 0: NVIDIA RTX 3050 Ti + +# Monitor resource usage +docker stats --no-stream + +# Expected: <500MB memory per service +``` + +--- + +### Kubernetes Deployment + +**Use Case**: Cloud deployment, horizontal scaling, high availability + +#### Step 1: Build and Push Images + +```bash +# Set registry +export REGISTRY=your-registry.com/foxhunt + +# Build and push all images +./deployment/build_docker_images.sh \ + --service all \ + --enable-gpu \ + --push \ + --registry $REGISTRY + +# Verify images pushed +docker images | grep $REGISTRY +``` + +#### Step 2: Create Kubernetes Manifests + +```bash +# Create namespace +kubectl create namespace foxhunt + +# Create secrets +kubectl create secret generic foxhunt-secrets -n foxhunt \ + --from-literal=database-url="postgresql://..." \ + --from-literal=jwt-secret="$(openssl rand -base64 64)" \ + --from-literal=benzinga-api-key="" \ + --from-literal=kill-switch-token="$(openssl rand -base64 32)" + +# Create ConfigMap +kubectl create configmap foxhunt-config -n foxhunt \ + --from-literal=rust-log=info \ + --from-literal=rate-limit-rps=100 +``` + +#### Step 3: Deploy Infrastructure + +```yaml +# k8s/infrastructure.yaml +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: foxhunt +spec: + selector: + app: postgres + ports: + - port: 5432 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres + namespace: foxhunt +spec: + serviceName: postgres + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: timescale/timescaledb:latest-pg16 + ports: + - containerPort: 5432 + env: + - name: POSTGRES_DB + value: foxhunt + - name: POSTGRES_USER + value: foxhunt + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: foxhunt-secrets + key: postgres-password + volumeMounts: + - name: postgres-data + mountPath: /var/lib/postgresql/data + volumeClaimTemplates: + - metadata: + name: postgres-data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 100Gi +``` + +```bash +# Apply infrastructure +kubectl apply -f k8s/infrastructure.yaml + +# Verify +kubectl get pods -n foxhunt +``` + +#### Step 4: Deploy Application Services + +```yaml +# k8s/trading-service.yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trading-service + namespace: foxhunt +spec: + replicas: 3 + selector: + matchLabels: + app: trading-service + template: + metadata: + labels: + app: trading-service + spec: + containers: + - name: trading-service + image: your-registry.com/foxhunt/trading-service:latest + ports: + - containerPort: 50051 + name: grpc + - containerPort: 9092 + name: metrics + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: foxhunt-secrets + key: database-url + - name: REDIS_URL + value: redis://redis:6379 + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: foxhunt-secrets + key: jwt-secret + resources: + requests: + memory: "512Mi" + cpu: "1000m" + limits: + memory: "2Gi" + cpu: "2000m" + livenessProbe: + exec: + command: ["/usr/local/bin/grpc_health_probe", "-addr=:50051"] + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + exec: + command: ["/usr/local/bin/grpc_health_probe", "-addr=:50051"] + initialDelaySeconds: 15 + periodSeconds: 5 +--- +apiVersion: v1 +kind: Service +metadata: + name: trading-service + namespace: foxhunt +spec: + selector: + app: trading-service + ports: + - name: grpc + port: 50051 + targetPort: 50051 + - name: metrics + port: 9092 + targetPort: 9092 +``` + +```bash +# Deploy services +kubectl apply -f k8s/trading-service.yaml +kubectl apply -f k8s/backtesting-service.yaml +kubectl apply -f k8s/ml-training-service.yaml +kubectl apply -f k8s/api-gateway.yaml + +# Verify +kubectl get pods -n foxhunt -l app=trading-service +kubectl logs -n foxhunt -l app=trading-service +``` + +#### Step 5: Configure Ingress + +```yaml +# k8s/ingress.yaml +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: foxhunt-ingress + namespace: foxhunt + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-prod" + nginx.ingress.kubernetes.io/backend-protocol: "GRPC" + nginx.ingress.kubernetes.io/ssl-redirect: "true" +spec: + ingressClassName: nginx + tls: + - hosts: + - api.foxhunt.trading + secretName: foxhunt-tls + rules: + - host: api.foxhunt.trading + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: api-gateway + port: + number: 50051 +``` + +```bash +# Apply ingress +kubectl apply -f k8s/ingress.yaml + +# Get external IP +kubectl get ingress -n foxhunt foxhunt-ingress +``` + +#### Step 6: Set Up Auto-Scaling + +```yaml +# k8s/hpa.yaml +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: trading-service-hpa + namespace: foxhunt +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: trading-service + minReplicas: 3 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 +``` + +```bash +# Apply HPA +kubectl apply -f k8s/hpa.yaml + +# Monitor scaling +kubectl get hpa -n foxhunt -w +``` + +--- + +## Rolling Updates + +### Zero-Downtime Update Procedure + +#### Strategy: Blue-Green Deployment + +```bash +# Step 1: Build new version +export NEW_VERSION=v1.1.0 +./deployment/build_release.sh --version $NEW_VERSION + +# Step 2: Deploy to "green" environment (parallel to "blue") +# Bare-metal: Deploy to separate servers +# Docker: Use docker-compose with different project name +# Kubernetes: Deploy to separate namespace + +# Step 3: Run smoke tests on green +./deployment/health_check.sh --target green --mode comprehensive + +# Step 4: Switch traffic (load balancer/DNS) +# Update load balancer backend to green environment + +# Step 5: Monitor for errors (rollback window: 15 minutes) +watch -n 5 'kubectl get pods -n foxhunt-green' +watch -n 5 'curl http://green.foxhunt.trading/metrics | grep error_rate' + +# Step 6: If successful, decommission blue environment +# If errors, rollback to blue +``` + +#### Rollback Procedure + +```bash +# Immediate rollback (switch load balancer back to blue) +# Total time: <30 seconds + +# Kubernetes rollback +kubectl rollout undo deployment/trading-service -n foxhunt +kubectl rollout status deployment/trading-service -n foxhunt + +# SystemD rollback +sudo systemctl stop foxhunt-trading +sudo cp /opt/foxhunt/bin/trading_service.backup /opt/foxhunt/bin/trading_service +sudo systemctl start foxhunt-trading + +# Docker rollback +docker-compose pull trading_service:previous +docker-compose up -d trading_service +``` + +### Database Migration Sequencing + +**Critical**: Database migrations must be backward-compatible for zero-downtime deployments. + +```bash +# Step 1: Create backward-compatible migration +# migrations/019_add_new_column.sql +BEGIN; + -- Add column with default value (safe) + ALTER TABLE orders ADD COLUMN new_field VARCHAR(255) DEFAULT 'default_value'; +COMMIT; + +# Step 2: Apply migration (old code still works) +cargo sqlx migrate run + +# Step 3: Deploy new application code +# (old code ignores new column, new code uses it) + +# Step 4: After deployment stable, remove default +# migrations/020_remove_default.sql +BEGIN; + ALTER TABLE orders ALTER COLUMN new_field DROP DEFAULT; +COMMIT; +``` + +--- + +## Scaling Procedures + +### Horizontal Scaling (Adding Instances) + +#### Stateless Services (Trading, Backtesting, API Gateway) + +**Bare-Metal**: +```bash +# Add new server to load balancer pool +# Install service on new server (same as initial deployment) +# Start service +sudo systemctl start foxhunt-trading + +# Verify health before adding to pool +curl http://new-server:9092/metrics + +# Add to load balancer +# Update Nginx upstream configuration +# nginx/upstream.conf +upstream trading_service { + server trading-1.foxhunt.internal:50051; + server trading-2.foxhunt.internal:50051; + server trading-3.foxhunt.internal:50051; # New server +} + +# Reload Nginx +sudo nginx -s reload +``` + +**Kubernetes**: +```bash +# Scale deployment +kubectl scale deployment trading-service -n foxhunt --replicas=5 + +# Or use HPA (automatic scaling) +kubectl autoscale deployment trading-service -n foxhunt \ + --min=3 --max=10 --cpu-percent=70 +``` + +**Docker Swarm**: +```bash +# Scale service +docker service scale foxhunt_trading_service=5 + +# Verify +docker service ps foxhunt_trading_service +``` + +#### Stateful Services (PostgreSQL, Redis) + +**PostgreSQL Read Replicas**: +```bash +# Create read replica +pg_basebackup -h primary.foxhunt.internal -D /var/lib/postgresql/replica \ + -U replication -P -v -R + +# Start replica +sudo systemctl start postgresql-replica + +# Update connection pool to use replica for reads +# services/trading_service/src/config.rs +DatabaseConfig { + write_url: "postgresql://primary:5432/foxhunt", + read_urls: vec![ + "postgresql://replica-1:5432/foxhunt", + "postgresql://replica-2:5432/foxhunt", + ], +} +``` + +**Redis Cluster**: +```bash +# Create 6-node cluster (3 masters, 3 replicas) +redis-cli --cluster create \ + redis-1:6379 redis-2:6379 redis-3:6379 \ + redis-4:6379 redis-5:6379 redis-6:6379 \ + --cluster-replicas 1 + +# Verify cluster +redis-cli -c cluster nodes +``` + +### Vertical Scaling (Resource Adjustments) + +#### Increase CPU/Memory (Kubernetes) + +```bash +# Update deployment resources +kubectl patch deployment trading-service -n foxhunt -p \ + '{"spec":{"template":{"spec":{"containers":[{"name":"trading-service","resources":{"requests":{"cpu":"2000m","memory":"4Gi"},"limits":{"cpu":"4000m","memory":"8Gi"}}}]}}}}' + +# Verify change +kubectl get deployment trading-service -n foxhunt -o yaml | grep -A 5 resources +``` + +#### Increase Disk (Bare-Metal) + +```bash +# Resize LVM volume (PostgreSQL data) +sudo lvextend -L +100G /dev/vg0/postgres +sudo resize2fs /dev/vg0/postgres + +# Verify +df -h /var/lib/postgresql +``` + +### Auto-Scaling Policies + +#### CPU-Based Scaling (Kubernetes) + +```yaml +# Already configured in HPA (see Kubernetes Deployment) +# Scales from 3 to 10 replicas based on 70% CPU utilization +``` + +#### Custom Metrics Scaling + +```yaml +# k8s/hpa-custom.yaml +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: trading-service-hpa + namespace: foxhunt +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: trading-service + minReplicas: 3 + maxReplicas: 20 + metrics: + # CPU + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + # Memory + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 + # Custom: Order processing latency + - type: Pods + pods: + metric: + name: order_processing_latency_p99_seconds + target: + type: AverageValue + averageValue: "0.05" # 50ms + # Custom: Queue depth + - type: Pods + pods: + metric: + name: order_queue_depth + target: + type: AverageValue + averageValue: "100" + behavior: + scaleDown: + stabilizationWindowSeconds: 300 # 5 minute cooldown + policies: + - type: Percent + value: 50 + periodSeconds: 60 + scaleUp: + stabilizationWindowSeconds: 0 + policies: + - type: Percent + value: 100 + periodSeconds: 15 + - type: Pods + value: 4 + periodSeconds: 15 + selectPolicy: Max +``` + +--- + +## Disaster Recovery + +### Recovery Time Objectives (RTO) and Recovery Point Objectives (RPO) + +| Component | RTO | RPO | Current Status | +|-----------|-----|-----|----------------| +| Database | <5 min | <1 min | ⚠️ Streaming replication needed | +| Trading Service | <1 min | 0 (stateless) | βœ… Ready (stateless) | +| ML Training Service | <2 min | <5 min | ⚠️ Checkpoint recovery needed | +| Risk Engine | <30 sec | 0 (real-time) | βœ… Ready (stateless) | +| Market Data | <10 sec | 0 (real-time) | ⚠️ Feed switching needed | +| API Gateway | <1 min | 0 (stateless) | βœ… Ready (stateless) | + +### Scenario 1: Database Failure + +**Impact**: Complete data persistence loss +**Recovery**: <5 minutes + +#### Detection +```bash +# Automatic: PostgreSQL connection pool alerts +# Manual: Check database connectivity +psql $DATABASE_URL -c "SELECT 1;" || echo "Database down!" +``` + +#### Recovery Procedure + +```bash +# Step 1: Promote read replica to primary (if available) +# On replica server: +sudo systemctl stop postgresql-replica +sudo -u postgres /usr/lib/postgresql/16/bin/pg_ctl promote -D /var/lib/postgresql/16/main + +# Verify promotion +psql -h replica-server -c "SELECT pg_is_in_recovery();" +# Expected: f (false = not in recovery = primary) + +# Step 2: Update connection strings +export OLD_PRIMARY=primary.foxhunt.internal +export NEW_PRIMARY=replica-1.foxhunt.internal + +# Update environment variables (all services) +sudo systemctl set-environment DATABASE_URL=postgresql://foxhunt:password@$NEW_PRIMARY:5432/foxhunt + +# Restart services to pick up new connection +sudo systemctl restart foxhunt-* + +# Step 3: Verify data consistency +psql -h $NEW_PRIMARY -c "SELECT MAX(created_at) FROM orders;" +psql -h $NEW_PRIMARY -c "SELECT count(*) FROM positions;" + +# Step 4: Create new replica from promoted primary +# (See Horizontal Scaling - PostgreSQL Read Replicas) +``` + +#### Backup Restoration (if no replica available) + +```bash +# Step 1: Restore from backup +# Backups location: S3 bucket foxhunt-backups/postgres/ +aws s3 cp s3://foxhunt-backups/postgres/latest.dump /tmp/postgres-restore.dump + +# Step 2: Create new database +sudo -u postgres createdb foxhunt_restored + +# Step 3: Restore data +pg_restore -h localhost -U foxhunt -d foxhunt_restored -v /tmp/postgres-restore.dump + +# Step 4: Rename databases (quick switchover) +sudo -u postgres psql < +sudo systemctl start foxhunt-trading + +# Issue: Database connection failure +# Verify database is running +sudo systemctl status postgresql + +# Issue: Corrupted cache/state +sudo rm -rf /tmp/foxhunt/* +sudo systemctl start foxhunt-trading + +# Step 4: If service still fails, rollback +sudo systemctl stop foxhunt-trading +sudo cp /opt/foxhunt/bin/trading_service.backup /opt/foxhunt/bin/trading_service +sudo systemctl start foxhunt-trading +``` + +### Scenario 3: Network Partition (Split Brain) + +**Impact**: Service isolation, data inconsistency risk +**Recovery**: <3 minutes + +#### Detection +```bash +# Services detect partition via health checks +# Logs will show: "Network partition detected - entering read-only mode" + +# Manual detection +for service in trading backtesting ml_training; do + curl http://$service.foxhunt.internal:9092/metrics | grep network_partition_active +done +``` + +#### Recovery Procedure + +```bash +# Step 1: Services automatically enter read-only mode +# No trading operations performed during partition + +# Step 2: Restore network connectivity +# Fix network infrastructure (routers, switches, firewalls) + +# Step 3: Verify connectivity restored +for service in trading-1 trading-2 trading-3; do + ping -c 3 $service.foxhunt.internal +done + +# Step 4: Services automatically reconcile state +# PostgreSQL replication catches up +# Redis cluster nodes rejoin + +# Step 5: Verify data consistency +psql -h primary -c "SELECT count(*) FROM orders;" > /tmp/primary_count +psql -h replica -c "SELECT count(*) FROM orders;" > /tmp/replica_count +diff /tmp/primary_count /tmp/replica_count +# Expected: No difference + +# Step 6: Resume normal operations +# Services automatically exit read-only mode after health checks pass +``` + +### Scenario 4: Complete System Failure (Data Center Outage) + +**Impact**: All services offline +**Recovery**: <15 minutes (cold start) + +#### Recovery Procedure + +```bash +# Step 1: Restore infrastructure +# Power on servers +# Verify network connectivity + +# Step 2: Start infrastructure services +sudo systemctl start postgresql +sudo systemctl start redis +sudo systemctl start vault + +# Wait for health +sleep 30 + +# Step 3: Unseal Vault (if using production seal) +export VAULT_ADDR=http://localhost:8200 +vault operator unseal +vault operator unseal +vault operator unseal + +# Verify unsealed +vault status + +# Step 4: Start application services (in order) +sudo systemctl start foxhunt-trading +sudo systemctl start foxhunt-backtesting +sudo systemctl start foxhunt-ml-training +sleep 30 +sudo systemctl start foxhunt-api-gateway + +# Step 5: Verify full system health +./deployment/health_check.sh --mode comprehensive + +# Step 6: Run data integrity checks +./deployment/scripts/data-integrity-check.sh + +# Step 7: Resume trading operations +# Notify operations team +# Monitor for anomalies for 1 hour +``` + +### Backup Procedures + +#### Database Backups + +```bash +# Automated daily backup (cron job) +# /etc/cron.daily/foxhunt-backup + +#!/bin/bash +set -euo pipefail + +BACKUP_DIR=/var/backups/foxhunt +S3_BUCKET=s3://foxhunt-backups/postgres +DATE=$(date +%Y%m%d_%H%M%S) + +# Create backup +pg_dump -h localhost -U foxhunt -Fc foxhunt > $BACKUP_DIR/foxhunt_$DATE.dump + +# Compress +gzip $BACKUP_DIR/foxhunt_$DATE.dump + +# Upload to S3 +aws s3 cp $BACKUP_DIR/foxhunt_$DATE.dump.gz $S3_BUCKET/ + +# Keep local backups for 7 days +find $BACKUP_DIR -name "*.dump.gz" -mtime +7 -delete + +# Verify backup +pg_restore --list $BACKUP_DIR/foxhunt_$DATE.dump.gz > /dev/null +``` + +#### Configuration Backups + +```bash +# Backup configurations to S3 daily +#!/bin/bash +S3_BUCKET=s3://foxhunt-backups/config +DATE=$(date +%Y%m%d) + +# Backup configuration files +tar -czf /tmp/foxhunt-config-$DATE.tar.gz /opt/foxhunt/config + +# Upload to S3 +aws s3 cp /tmp/foxhunt-config-$DATE.tar.gz $S3_BUCKET/ + +# Cleanup +rm /tmp/foxhunt-config-$DATE.tar.gz +``` + +#### Model Backups + +```bash +# ML models are already stored in S3 (foxhunt-models bucket) +# Checkpoints stored in S3 (foxhunt-checkpoints bucket) +# Retention: Latest 10 checkpoints per model + +# Verify model backups +aws s3 ls s3://foxhunt-models/ --recursive | grep -E "mamba2|dqn|ppo|tft" + +# Verify checkpoint backups +aws s3 ls s3://foxhunt-checkpoints/ --recursive | tail -10 +``` + +--- + +## Monitoring and Alerting + +### Critical Metrics + +#### System Health Metrics + +| Metric | Warning Threshold | Critical Threshold | Action | +|--------|-------------------|-------------------|--------| +| Order latency (p99) | >50ΞΌs | >100ΞΌs | Scale horizontally | +| Error rate | >0.1% | >1% | Page on-call engineer | +| Memory usage | >80% | >90% | Restart service, add memory | +| CPU usage | >70% | >90% | Scale horizontally | +| Disk usage | >80% | >90% | Expand disk, archive data | +| Database connections | >80% | >95% | Increase pool size | +| Queue depth | >1000 | >5000 | Scale workers | + +#### Grafana Dashboards + +**Access**: http://localhost:3000 (admin / foxhunt123) + +**Pre-configured Dashboards**: +1. **HFT Trading Performance** + - Order latency (p50, p95, p99) + - Throughput (orders/second) + - Error rates by error type + - Position counts + +2. **System Resources** + - CPU usage per service + - Memory usage per service + - Disk I/O + - Network bandwidth + +3. **Database Performance** + - Connection pool utilization + - Query latency + - Active queries + - Replication lag + +4. **ML Model Performance** + - Inference latency + - Model accuracy + - Cache hit rate + - GPU utilization + +#### Prometheus Alerts + +**Alert Rules** (`config/prometheus/rules/alerts.yml`): + +```yaml +groups: +- name: foxhunt_critical + interval: 30s + rules: + # Trading latency + - alert: HighOrderLatency + expr: histogram_quantile(0.99, rate(order_processing_duration_seconds_bucket[5m])) > 0.0001 + for: 2m + labels: + severity: critical + annotations: + summary: "High order processing latency" + description: "P99 latency is {{ $value }}s (threshold: 100ΞΌs)" + + # Error rate + - alert: HighErrorRate + expr: rate(foxhunt_errors_total[5m]) > 0.01 + for: 1m + labels: + severity: critical + annotations: + summary: "High error rate detected" + description: "Error rate is {{ $value }} (threshold: 1%)" + + # Service down + - alert: ServiceDown + expr: up{job=~"foxhunt.*"} == 0 + for: 30s + labels: + severity: critical + annotations: + summary: "Service {{ $labels.job }} is down" + description: "Service has been down for 30 seconds" + + # Database connections + - alert: HighDatabaseConnections + expr: pg_stat_database_numbackends / pg_settings_max_connections > 0.8 + for: 5m + labels: + severity: warning + annotations: + summary: "High database connection usage" + description: "{{ $value | humanizePercentage }} of max connections in use" + + # Disk usage + - alert: HighDiskUsage + expr: (node_filesystem_size_bytes - node_filesystem_free_bytes) / node_filesystem_size_bytes > 0.9 + for: 5m + labels: + severity: critical + annotations: + summary: "High disk usage on {{ $labels.instance }}" + description: "Disk usage is {{ $value | humanizePercentage }}" + + # GPU utilization (ML services) + - alert: GPUNotDetected + expr: nvidia_gpu_count == 0 + for: 1m + labels: + severity: warning + annotations: + summary: "GPU not detected in ML service" + description: "ML service running without GPU acceleration" +``` + +#### Alert Routing (AlertManager) + +**Configuration** (`config/prometheus/alertmanager.yml`): + +```yaml +global: + resolve_timeout: 5m + +route: + group_by: ['alertname', 'cluster', 'service'] + group_wait: 10s + group_interval: 10s + repeat_interval: 12h + receiver: 'default' + routes: + # Critical alerts -> Page on-call + - match: + severity: critical + receiver: 'pagerduty' + continue: true + + # Warning alerts -> Slack + - match: + severity: warning + receiver: 'slack' + +receivers: +- name: 'default' + email_configs: + - to: 'ops@foxhunt.trading' + from: 'alerts@foxhunt.trading' + smarthost: 'smtp.gmail.com:587' + auth_username: 'alerts@foxhunt.trading' + auth_password: '' + +- name: 'pagerduty' + pagerduty_configs: + - service_key: '' + description: '{{ .GroupLabels.alertname }}: {{ .Annotations.summary }}' + +- name: 'slack' + slack_configs: + - api_url: '' + channel: '#foxhunt-alerts' + title: '{{ .GroupLabels.alertname }}' + text: '{{ .Annotations.description }}' +``` + +#### Log Aggregation + +**Locations**: +- **SystemD logs**: `journalctl -u foxhunt-* -f` +- **Application logs**: `/opt/foxhunt/logs/*.log` +- **Docker logs**: `docker-compose logs -f` + +**Log Levels**: +- **ERROR**: Failures requiring immediate attention +- **WARN**: Potential issues, degraded performance +- **INFO**: Normal operations, state changes +- **DEBUG**: Detailed diagnostics (disabled in production) + +**Important Log Patterns**: + +```bash +# Monitor errors in real-time +journalctl -u foxhunt-* -f | grep ERROR + +# Check for panics +journalctl -u foxhunt-* --since "1 hour ago" | grep -i panic + +# Monitor database connection issues +journalctl -u foxhunt-trading -f | grep "database connection" + +# Check for configuration errors (Agent 96 finding) +journalctl -u foxhunt-* | grep "Configuration error" +``` + +--- + +## Troubleshooting + +### Common Issues and Solutions + +#### Issue 1: Service Won't Start + +**Agent 96 Finding**: Backtesting service failed due to missing `BENZINGA_API_KEY`. + +**Symptoms**: +```bash +sudo systemctl status foxhunt-trading +# Output: failed (code=exited, status=1) +``` + +**Diagnosis**: +```bash +# Check logs for error +sudo journalctl -u foxhunt-trading -n 50 --no-pager + +# Common errors: +# - "Configuration error in field 'api_key'" +# - "Failed to connect to database" +# - "Port already in use" +# - "Failed to load JWT secret" +``` + +**Solutions**: + +**Missing Environment Variable**: +```bash +# Check environment variables +sudo systemctl show foxhunt-trading | grep Environment + +# Add missing variable +sudo systemctl edit foxhunt-trading +# Add in override: +[Service] +Environment="BENZINGA_API_KEY=" + +# Restart +sudo systemctl restart foxhunt-trading +``` + +**Port Already in Use**: +```bash +# Find process using port +sudo lsof -i :50052 + +# Kill process +sudo kill -9 + +# Restart service +sudo systemctl start foxhunt-trading +``` + +**Database Connection Failure**: +```bash +# Verify database is running +sudo systemctl status postgresql + +# Test connection +psql $DATABASE_URL -c "SELECT 1;" + +# Check connection pool settings +# trading_service/src/config.rs +# Increase pool size if needed +``` + +**JWT Secret Validation Failure**: +```bash +# Issue: JWT secret doesn't meet requirements (64+ chars, mixed case, symbols) +# Generate new secret +openssl rand -base64 64 | tr -d '\n' + +# Update environment variable +export JWT_SECRET="" + +# Restart service +sudo systemctl restart foxhunt-trading +``` + +#### Issue 2: High Latency + +**Symptoms**: +```bash +# Prometheus metric shows high latency +curl http://localhost:9092/metrics | grep order_processing_duration_seconds + +# Grafana dashboard shows p99 >100ΞΌs +``` + +**Diagnosis**: +```bash +# Check CPU frequency scaling (should be performance mode for HFT) +cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor + +# Check CPU affinity +ps -eLo pid,tid,psr,comm | grep trading_service + +# Check network latency +ping -c 10 + +# Check database query performance +psql $DATABASE_URL -c "SELECT query, mean_exec_time FROM pg_stat_statements ORDER BY mean_exec_time DESC LIMIT 10;" +``` + +**Solutions**: + +**CPU Frequency Scaling**: +```bash +# Set performance governor +sudo cpupower frequency-set -g performance + +# Verify +cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +# Expected: performance +``` + +**CPU Affinity**: +```bash +# Pin trading service to specific cores (avoid context switching) +sudo systemctl edit foxhunt-trading +# Add: +[Service] +CPUAffinity=0-7 # Use cores 0-7 exclusively + +# Restart +sudo systemctl restart foxhunt-trading +``` + +**Database Query Optimization**: +```bash +# Add indexes for slow queries +psql $DATABASE_URL < GPU Requirements section +``` + +**Update Dockerfile** (Agent 96 fix): +```dockerfile +# Change base image to include CUDA runtime +FROM nvidia/cuda:12.3.0-runtime-ubuntu22.04 as runtime + +# Install CUDA libraries +RUN apt-get update && apt-get install -y \ + cuda-runtime-12-3 \ + && rm -rf /var/lib/apt/lists/* +``` + +**Update docker-compose.yml**: +```yaml +ml_training_service: + runtime: nvidia + environment: + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=compute,utility +``` + +**Rebuild and restart**: +```bash +docker-compose build ml_training_service +docker-compose up -d ml_training_service + +# Verify GPU access +docker-compose exec ml_training_service nvidia-smi +``` + +#### Issue 5: Configuration Errors + +**Agent 96 Finding**: Missing configuration values cause service startup failures. + +**Symptoms**: +```bash +# Service logs show configuration error +journalctl -u foxhunt-backtesting | grep "Configuration error" +# Output: "Configuration error in field 'api_key': Benzinga API key is required" +``` + +**Diagnosis**: +```bash +# Check all environment variables +sudo systemctl show foxhunt-backtesting | grep Environment + +# Validate configuration +/opt/foxhunt/bin/backtesting_service config validate +``` + +**Solutions**: + +**Add Missing Variables**: +```bash +# Create environment file +sudo vim /etc/foxhunt/backtesting.env + +# Add: +BENZINGA_API_KEY= +DATABASE_URL=postgresql://... +REDIS_URL=redis://... + +# Update SystemD service +sudo systemctl edit foxhunt-backtesting +[Service] +EnvironmentFile=/etc/foxhunt/backtesting.env + +# Restart +sudo systemctl restart foxhunt-backtesting +``` + +**Validate All Services**: +```bash +# Run configuration validation for all services +for service in trading backtesting ml_training api_gateway; do + echo "Validating $service..." + /opt/foxhunt/bin/${service}_service config validate +done +``` + +#### Issue 6: Database Connection Pool Exhausted + +**Symptoms**: +```bash +# Error in logs +journalctl -u foxhunt-trading | grep "connection pool" +# Output: "Failed to acquire connection from pool: timeout" +``` + +**Diagnosis**: +```bash +# Check active connections +psql $DATABASE_URL -c "SELECT count(*) FROM pg_stat_activity;" + +# Check max connections +psql $DATABASE_URL -c "SHOW max_connections;" + +# Check connection pool configuration +# services/trading_service/src/config.rs +``` + +**Solutions**: + +**Increase Pool Size**: +```bash +# Edit configuration +# services/trading_service/src/config.rs +DatabaseConfig { + max_connections: 100, // Increase from default (20) + min_connections: 10, + connection_timeout: Duration::from_secs(30), +} + +# Rebuild and restart +cargo build --release -p trading_service +sudo systemctl restart foxhunt-trading +``` + +**Increase PostgreSQL Max Connections**: +```bash +# Edit postgresql.conf +sudo vim /etc/postgresql/16/main/postgresql.conf + +# Change: +max_connections = 200 # Increase from 100 + +# Restart PostgreSQL +sudo systemctl restart postgresql +``` + +**Fix Connection Leaks**: +```bash +# Audit code for connections not being returned to pool +# Ensure all database operations use connection pooling + +# Check for long-running queries +psql $DATABASE_URL -c "SELECT pid, now() - query_start as duration, query FROM pg_stat_activity WHERE state = 'active' ORDER BY duration DESC;" + +# Kill long-running queries if needed +psql $DATABASE_URL -c "SELECT pg_terminate_backend();" +``` + +--- + +## Security Procedures + +### JWT Secret Rotation + +**Frequency**: Every 90 days or after suspected compromise + +**Procedure**: +```bash +# Step 1: Generate new secret +NEW_JWT_SECRET=$(openssl rand -base64 64 | tr -d '\n') + +# Step 2: Store in Vault +vault kv put secret/foxhunt/jwt \ + secret="$NEW_JWT_SECRET" \ + previous_secret="$OLD_JWT_SECRET" # Keep for transition period + +# Step 3: Update services with dual-key support +# Services will accept both old and new tokens during transition + +# Step 4: After 24 hours (all clients refreshed), remove old secret +vault kv put secret/foxhunt/jwt \ + secret="$NEW_JWT_SECRET" + +# Step 5: Verify all clients using new secret +curl http://localhost:9091/metrics | grep jwt_validation_success_total +``` + +### Vault Token Management + +**Procedure**: +```bash +# Step 1: Rotate AppRole secret +vault write -f auth/approle/role/foxhunt-services/secret-id + +# Step 2: Get new secret ID +NEW_SECRET_ID=$(vault write -f -format=json auth/approle/role/foxhunt-services/secret-id | jq -r .data.secret_id) + +# Step 3: Update services +sudo systemctl set-environment VAULT_SECRET_ID="$NEW_SECRET_ID" + +# Step 4: Restart services to pick up new secret +sudo systemctl restart foxhunt-* + +# Step 5: Revoke old secret (after verifying new one works) +vault write auth/approle/role/foxhunt-services/secret-id-accessor/ destroy=true +``` + +### TLS Certificate Renewal + +**Frequency**: Every 90 days (Let's Encrypt) or per certificate lifetime + +**Procedure**: +```bash +# Using cert-manager (Kubernetes) +# Automatic renewal - no action needed + +# Using Let's Encrypt (bare-metal) +sudo certbot renew --nginx + +# Manual renewal +sudo certbot certonly --nginx -d api.foxhunt.trading + +# Reload Nginx +sudo nginx -s reload + +# Verify new certificate +echo | openssl s_client -servername api.foxhunt.trading -connect api.foxhunt.trading:443 2>/dev/null | openssl x509 -noout -dates +``` + +### Audit Log Review + +**Frequency**: Daily (automated), detailed review weekly + +**Procedure**: +```bash +# Automated daily check for suspicious activity +cat <<'EOF' > /etc/cron.daily/foxhunt-audit-check +#!/bin/bash +set -euo pipefail + +AUDIT_LOG=/var/log/foxhunt/audit.log +REPORT=/tmp/audit-report-$(date +%Y%m%d).txt + +# Check for failed authentication attempts +echo "Failed auth attempts:" >> $REPORT +grep "authentication failed" $AUDIT_LOG | wc -l >> $REPORT + +# Check for unauthorized access attempts +echo "Unauthorized access:" >> $REPORT +grep "unauthorized" $AUDIT_LOG | wc -l >> $REPORT + +# Check for privileged operations +echo "Privileged operations:" >> $REPORT +grep "kill_switch activated" $AUDIT_LOG >> $REPORT + +# Email report +mail -s "Foxhunt Daily Audit Report" ops@foxhunt.trading < $REPORT +EOF + +chmod +x /etc/cron.daily/foxhunt-audit-check +``` + +### Security Incident Response + +**Detection**: +```bash +# Automated alerts from Prometheus/AlertManager +# Manual review of logs +# External security notifications +``` + +**Immediate Response** (first 15 minutes): +```bash +# Step 1: Activate kill switch (halt all trading) +curl -X POST http://localhost:8080/emergency/kill \ + -H "X-Kill-Switch-Token: $KILL_SWITCH_MASTER_TOKEN" + +# Step 2: Isolate affected services +sudo systemctl stop foxhunt-trading +sudo systemctl stop foxhunt-api-gateway + +# Step 3: Enable read-only mode for remaining services +curl -X POST http://localhost:8082/mode/readonly + +# Step 4: Capture forensic data +./deployment/scripts/capture-forensics.sh > /var/log/foxhunt/incident-$(date +%s).log + +# Step 5: Notify security team +# Send alert with incident details +``` + +**Investigation** (next 1-4 hours): +```bash +# Review audit logs +grep "$(date +%Y-%m-%d)" /var/log/foxhunt/audit.log | grep -E "(failed|unauthorized|suspicious)" + +# Check for unauthorized access +psql $DATABASE_URL -c "SELECT * FROM audit_logs WHERE action = 'unauthorized' AND created_at > NOW() - INTERVAL '24 hours';" + +# Review network connections +sudo netstat -anp | grep ESTABLISHED | grep trading_service + +# Check for malware/rootkits +sudo rkhunter --check +sudo chkrootkit +``` + +**Remediation**: +```bash +# Rotate all secrets (see above procedures) +# Apply security patches +# Update firewall rules if needed +# Review and update security policies +``` + +**Recovery**: +```bash +# After remediation complete, resume operations +curl -X POST http://localhost:8080/trading/resume \ + -H "X-Kill-Switch-Token: $KILL_SWITCH_MASTER_TOKEN" + +# Monitor closely for 24 hours +watch -n 60 './deployment/health_check.sh --mode comprehensive' +``` + +--- + +## Appendix A: Service Ports Reference + +| Service | External Port | Internal Port | Metrics Port | Health Port | +|---------|--------------|---------------|--------------|-------------| +| PostgreSQL | 5432 | 5432 | - | - | +| Redis | 6379 | 6379 | - | - | +| Vault | 8200 | 8200 | - | - | +| InfluxDB | 8086 | 8086 | - | - | +| Prometheus | 9090 | 9090 | - | - | +| Grafana | 3000 | 3000 | - | - | +| Trading Service | 50052 | 50051 | 9092 | 8080 | +| Backtesting Service | 50053 | 50052 | 9093 | - | +| ML Training Service | 50054 | 50053 | 9094 | - | +| API Gateway | 50051 | 50050 | 9091 | - | + +## Appendix B: Environment Variables Reference + +See [Prerequisites -> Environment Variables](#2-environment-variables) for complete reference. + +## Appendix C: Commands Quick Reference + +```bash +# Start all services +docker-compose up -d + +# Stop all services +docker-compose down + +# View logs +docker-compose logs -f +journalctl -u foxhunt- -f + +# Health check +./deployment/health_check.sh --mode comprehensive + +# Restart service +sudo systemctl restart foxhunt- +docker-compose restart + +# Check metrics +curl http://localhost:9092/metrics + +# Database connection +psql $DATABASE_URL + +# Vault login +export VAULT_ADDR=http://localhost:8200 +vault login +``` + +--- + +**Document Version**: 1.0.0 +**Last Updated**: 2025-10-07 +**Next Review**: After Wave 125 Phase 3B completion +**Maintainer**: DevOps Team +**Contact**: ops@foxhunt.trading diff --git a/QUICK_START_PRODUCTION.md b/QUICK_START_PRODUCTION.md new file mode 100644 index 000000000..7bd0cc757 --- /dev/null +++ b/QUICK_START_PRODUCTION.md @@ -0,0 +1,408 @@ +# Quick Start Production Deployment - Foxhunt HFT + +**Time Required**: 15-20 minutes (Docker), 30-45 minutes (Bare-Metal) +**Prerequisites**: All infrastructure ready (PostgreSQL, Redis, Vault) + +--- + +## πŸš€ Docker Deployment (Recommended for Testing/Staging) + +### Step 1: Fix Agent 96 Issues (REQUIRED - 5 minutes) + +```bash +cd /opt/foxhunt + +# Fix 1: Update Dockerfile paths (all 4 services) +find services -name Dockerfile -exec sed -i 's|COPY crates/config|COPY config|g' {} \; + +# Fix 2: Add ML Training Service entry point +echo 'CMD ["ml_training_service", "serve"]' >> services/ml_training_service/Dockerfile + +# Fix 3: Set Benzinga API key +export BENZINGA_API_KEY="your-benzinga-pro-api-key" +echo "BENZINGA_API_KEY=$BENZINGA_API_KEY" >> .env +``` + +### Step 2: Configure Environment (2 minutes) + +```bash +# Generate required secrets +export JWT_SECRET=$(openssl rand -base64 64 | tr -d '\n') +export KILL_SWITCH_TOKEN=$(openssl rand -base64 32) + +# Create .env file +cat > .env < # Docker +sudo journalctl -u foxhunt- -n 50 # Bare-metal + +# Common fix: Restart +docker-compose restart +sudo systemctl restart foxhunt- +``` + +### Issue: Missing Environment Variable + +```bash +# Docker: Add to .env +echo "MISSING_VAR=value" >> .env +docker-compose up -d + +# Bare-metal: Add to SystemD +sudo systemctl edit foxhunt- +# Add: Environment="MISSING_VAR=value" +sudo systemctl restart foxhunt- +``` + +### Issue: GPU Not Detected + +```bash +# Docker: Install nvidia-docker2 +sudo apt-get install -y nvidia-docker2 +sudo systemctl restart docker + +# Verify +docker run --rm --gpus all nvidia/cuda:12.3.0-base-ubuntu22.04 nvidia-smi +``` + +### Issue: Port Already in Use + +```bash +# Find process +sudo lsof -i : + +# Kill process +sudo kill -9 + +# Restart service +docker-compose up -d +``` + +### Issue: Database Connection Failed + +```bash +# Verify PostgreSQL running +docker-compose ps postgres # Docker +sudo systemctl status postgresql # Bare-metal + +# Test connection +psql $DATABASE_URL -c "SELECT 1;" + +# Restart database +docker-compose restart postgres +sudo systemctl restart postgresql +``` + +--- + +## πŸ“Š Monitoring Access + +- **Grafana**: http://localhost:3000 (admin / foxhunt123) +- **Prometheus**: http://localhost:9090 +- **Metrics**: + - Trading: http://localhost:9092/metrics + - Backtesting: http://localhost:9093/metrics + - ML Training: http://localhost:9094/metrics + - API Gateway: http://localhost:9091/metrics + +--- + +## πŸ“ž Next Steps + +1. **Configure monitoring alerts**: Edit `config/prometheus/rules/alerts.yml` +2. **Set up backups**: Configure daily PostgreSQL backups to S3 +3. **Enable TLS**: Configure SSL/TLS certificates for production +4. **Review security**: Rotate default passwords and tokens +5. **Load testing**: Run `services/load_tests` to validate performance + +--- + +## πŸ“š Full Documentation + +- **Complete Runbook**: `PRODUCTION_DEPLOYMENT_RUNBOOK.md` +- **Emergency Procedures**: `EMERGENCY_PROCEDURES.md` +- **Maintenance Checklist**: `MAINTENANCE_CHECKLIST.md` +- **Architecture**: `CLAUDE.md` + +--- + +**Last Updated**: 2025-10-07 +**Version**: 1.0.0 +**Estimated Time**: 15-20 minutes (Docker), 30-45 minutes (Bare-Metal) diff --git a/SCALING_PLAYBOOK.md b/SCALING_PLAYBOOK.md new file mode 100644 index 000000000..504b0950f --- /dev/null +++ b/SCALING_PLAYBOOK.md @@ -0,0 +1,862 @@ +# Scaling Playbook - Foxhunt HFT Trading System + +**Last Updated**: 2025-10-07 +**Status**: Production Ready +**Audience**: DevOps, SRE, Platform Engineers + +--- + +## Table of Contents + +1. [Overview](#overview) +2. [When to Scale](#when-to-scale) +3. [Monitoring Metrics](#monitoring-metrics) +4. [Scaling Decisions](#scaling-decisions) +5. [Cost vs Performance Tradeoffs](#cost-vs-performance-tradeoffs) +6. [Scaling Procedures](#scaling-procedures) +7. [Troubleshooting](#troubleshooting) +8. [Emergency Response](#emergency-response) + +--- + +## Overview + +This playbook provides operational guidance for scaling the Foxhunt HFT Trading System. It covers: +- **When** to scale (metrics and thresholds) +- **How** to scale (manual and automated procedures) +- **Why** to scale (performance, cost, reliability) +- **What** to monitor (metrics and alerts) + +### Scaling Philosophy + +1. **Proactive Scaling**: Scale before problems occur (monitor trends) +2. **Cost-Aware Scaling**: Balance performance with cost +3. **Automated First**: Use HPA for routine scaling, manual for exceptions +4. **Graceful Degradation**: System should degrade gracefully under load +5. **Measured Response**: Base decisions on metrics, not gut feeling + +--- + +## When to Scale + +### Scale-Up Triggers (Add Capacity) + +#### Critical (Immediate Action Required) + +| Metric | Threshold | Action | Timeline | +|--------|-----------|--------|----------| +| P99 Latency | > 100ms | Scale up immediately | < 5 minutes | +| Error Rate | > 1% | Scale up immediately | < 5 minutes | +| CPU Utilization | > 90% | Scale up immediately | < 5 minutes | +| Memory Utilization | > 95% | Scale up immediately | < 2 minutes | +| Service Down | Any instance down | Replace/scale up | < 2 minutes | + +**Example Alert**: +``` +CRITICAL: API Gateway P99 Latency at 150ms (threshold: 100ms) +Current replicas: 3, CPU: 92%, Memory: 78% +Action: Scale up to 6 replicas immediately +``` + +#### Warning (Plan Scaling Within 1 Hour) + +| Metric | Threshold | Action | Timeline | +|--------|-----------|--------|----------| +| P99 Latency | > 50ms | Plan scale-up | < 1 hour | +| Error Rate | > 0.5% | Plan scale-up | < 1 hour | +| CPU Utilization | > 80% | Plan scale-up | < 1 hour | +| Memory Utilization | > 85% | Plan scale-up | < 1 hour | +| Request Queue Depth | > 100 | Plan scale-up | < 30 minutes | + +**Example Alert**: +``` +WARNING: Trading Service CPU at 85% (threshold: 80%) +Current replicas: 5, Request rate: 8K req/s +Action: Review traffic patterns, plan scale-up to 7 replicas +``` + +#### Proactive (Plan Scaling Within 24 Hours) + +| Metric | Threshold | Action | Timeline | +|--------|-----------|--------|----------| +| Traffic Growth | > 20% week-over-week | Plan capacity expansion | < 1 week | +| CPU Utilization Trend | Increasing 10%/week | Plan scale-up | < 1 week | +| Peak Traffic Approaching | Predicted > 80% capacity | Pre-scale before peak | < 24 hours | +| Scheduled Events | Known high-traffic events | Pre-scale 1 hour before | < 1 hour before event | + +**Example**: +``` +INFO: Traffic growing 25% week-over-week (5K β†’ 6.25K req/s) +Current capacity: 10K req/s (10 replicas) +Projected capacity need (4 weeks): 12.5K req/s +Action: Plan to add 2 replicas (12 total) within 2 weeks +``` + +### Scale-Down Triggers (Reduce Capacity) + +#### Safe to Scale Down + +| Metric | Threshold | Action | Timeline | +|--------|-----------|--------|----------| +| CPU Utilization | < 40% for 30 minutes | Scale down by 25% | < 10 minutes | +| Memory Utilization | < 50% for 30 minutes | Scale down by 25% | < 10 minutes | +| Request Rate | < 50% of capacity | Scale down by 25% | < 10 minutes | +| Off-Peak Hours | Night/weekend (known pattern) | Scale down to min replicas | Scheduled | + +**Example**: +``` +INFO: API Gateway CPU at 35% for 45 minutes (threshold: 40% for 30m) +Current replicas: 10, Request rate: 2K req/s (capacity: 20K req/s) +Action: Scale down to 8 replicas (save 20% cost, maintain 10K req/s capacity) +``` + +#### Never Scale Down + +- Error rate > 0.1% +- P99 latency > 20ms +- Recent scale-up (< 10 minutes ago) +- During peak traffic hours +- Less than minReplicas (HA requirement) + +--- + +## Monitoring Metrics + +### Service-Level Metrics (RED Metrics) + +**Request Rate**: +```promql +# Total request rate across all pods +sum(rate(grpc_requests_total{service="trading-service"}[5m])) + +# Per-pod request rate +rate(grpc_requests_total{service="trading-service"}[5m]) +``` + +**Error Rate**: +```promql +# Error percentage +sum(rate(grpc_requests_total{service="trading-service",status="error"}[5m])) / +sum(rate(grpc_requests_total{service="trading-service"}[5m])) * 100 + +# Alert if > 0.5% +``` + +**Latency (Duration)**: +```promql +# P99 latency +histogram_quantile(0.99, + rate(grpc_request_duration_seconds_bucket{service="trading-service"}[5m])) + +# P50, P95, P99.9 +histogram_quantile(0.50, ...) +histogram_quantile(0.95, ...) +histogram_quantile(0.999, ...) +``` + +### Resource Utilization Metrics + +**CPU**: +```promql +# Average CPU utilization across pods +avg(rate(container_cpu_usage_seconds_total{pod=~"trading-service.*"}[5m])) * 100 + +# Per-pod CPU +rate(container_cpu_usage_seconds_total{pod="trading-service-1"}[5m]) * 100 +``` + +**Memory**: +```promql +# Average memory utilization +avg(container_memory_working_set_bytes{pod=~"trading-service.*"}) / +avg(container_spec_memory_limit_bytes{pod=~"trading-service.*"}) * 100 + +# Memory usage in MB +container_memory_working_set_bytes{pod="trading-service-1"} / 1024 / 1024 +``` + +### Scaling Metrics + +**Replica Count**: +```promql +# Current replicas +kube_deployment_status_replicas{deployment="trading-service"} + +# Desired replicas (from HPA) +kube_horizontalpodautoscaler_status_desired_replicas{hpa="trading-service-hpa"} + +# Available replicas (healthy) +kube_deployment_status_replicas_available{deployment="trading-service"} +``` + +**Scaling Events**: +```bash +# View recent scaling events +kubectl get events -n foxhunt \ + --field-selector involvedObject.kind=HorizontalPodAutoscaler \ + --sort-by='.lastTimestamp' + +# Example output: +# 5m ago: ScaledUp from 5 to 7 replicas (CPU: 85%) +# 15m ago: ScaledDown from 10 to 8 replicas (CPU: 35%) +``` + +### Database Metrics + +**Connection Pool**: +```promql +# PostgreSQL active connections +pg_stat_database_numbackends{datname="foxhunt"} + +# Connection pool utilization +pg_stat_database_numbackends / pg_settings_max_connections * 100 + +# Alert if > 80% +``` + +**Replication Lag**: +```promql +# Replication lag (should be < 100ms) +pg_stat_replication_replay_lag_bytes + +# Convert to milliseconds +pg_stat_replication_replay_lag_seconds * 1000 +``` + +**Query Performance**: +```promql +# Slow queries (> 100ms) +pg_stat_statements_mean_exec_time_seconds{query=~".*"} > 0.1 + +# Top 10 slowest queries +topk(10, pg_stat_statements_mean_exec_time_seconds) +``` + +--- + +## Scaling Decisions + +### Decision Matrix + +| Symptom | Root Cause | Solution | Cost Impact | +|---------|-----------|----------|-------------| +| **High CPU (> 80%)** | Insufficient compute | Scale up replicas | **High** (+20% cost per replica) | +| **High Memory (> 90%)** | Memory leak or under-provisioned | Fix leak OR increase memory limits | **Medium** (10-20% cost increase) | +| **High Latency + Low CPU** | Database bottleneck | Add read replicas | **Medium** (+30% DB cost) | +| **High Error Rate** | Service overload | Scale up replicas | **High** (+20% cost per replica) | +| **Connection Exhaustion** | Too many service instances | Add PgBouncer | **Low** (<5% cost increase) | +| **Redis Memory Full** | Cache size exceeds capacity | Add Redis Cluster | **Medium** (+50% Redis cost) | +| **Queue Depth High** | Worker shortage | Scale up workers | **High** (+20% per worker) | +| **Disk I/O High** | Storage bottleneck | Upgrade disk (SSD β†’ NVMe) | **High** (2-3x storage cost) | + +### Decision Tree + +``` +Start: Is there a performance issue? + β”œβ”€ Yes: What metric is degraded? + β”‚ β”œβ”€ Latency (P99 > 100ms) + β”‚ β”‚ β”œβ”€ CPU high? β†’ Scale up replicas + β”‚ β”‚ β”œβ”€ Database slow? β†’ Add read replicas or optimize queries + β”‚ β”‚ └─ Network slow? β†’ Check load balancer, network config + β”‚ β”œβ”€ Error Rate (> 0.5%) + β”‚ β”‚ β”œβ”€ Service overload? β†’ Scale up replicas + β”‚ β”‚ β”œβ”€ Database errors? β†’ Check connection pool, add replicas + β”‚ β”‚ └─ External API errors? β†’ Implement retry, circuit breaker + β”‚ └─ Throughput (cannot handle load) + β”‚ β”œβ”€ CPU bottleneck? β†’ Scale up replicas + β”‚ β”œβ”€ Database bottleneck? β†’ Optimize queries, add replicas + β”‚ └─ Network bottleneck? β†’ Upgrade network, load balancer + └─ No: Is cost optimization possible? + β”œβ”€ Over-provisioned (CPU < 40%)? β†’ Scale down replicas + β”œβ”€ Off-peak hours? β†’ Scale down to minReplicas + └─ Right-size instances? β†’ Switch to smaller instance types +``` + +### Example Scenarios + +#### Scenario 1: Trading Service High CPU + +**Symptoms**: +- CPU: 92% (threshold: 80%) +- P99 Latency: 75ms (threshold: 100ms, target: 5ms) +- Error Rate: 0.3% (threshold: 0.5%) +- Current replicas: 5 +- Request rate: 10K orders/sec + +**Analysis**: +- CPU high but latency still acceptable +- Error rate below threshold +- Approaching capacity (10K orders/sec Γ· 5 = 2K orders/sec per pod) + +**Decision**: Scale up proactively +- Target CPU: 70% (with 20% headroom) +- Required replicas: 10K orders/sec Γ· (2K Γ— 0.7) = 7.14 β†’ 8 replicas +- Cost impact: +60% (5 β†’ 8 replicas) + +**Action**: +```bash +# Manual scale-up +kubectl scale deployment trading-service --replicas=8 -n foxhunt + +# Verify scaling +watch kubectl get pods -n foxhunt -l app=trading-service +``` + +#### Scenario 2: Database Connection Exhaustion + +**Symptoms**: +- Error: "FATAL: remaining connection slots are reserved" +- PostgreSQL connections: 95/100 +- Services running: API Gateway (3), Trading (10), Backtesting (5) +- Expected connections: (3 + 10 + 5) Γ— 10 = 180 (exceeds limit!) + +**Analysis**: +- Connection pool not configured +- Each service instance uses 10 connections +- Total connections exceed PostgreSQL limit + +**Decision**: Add PgBouncer connection pooler +- PgBouncer pools 180 connections β†’ 50 PostgreSQL connections +- No need to reduce service instances + +**Action**: +```bash +# Deploy PgBouncer +kubectl apply -f pgbouncer-deployment.yaml + +# Update service connection strings +DATABASE_URL=postgresql://foxhunt:password@pgbouncer:6432/foxhunt +``` + +#### Scenario 3: Pre-Scaling for Known Event + +**Symptoms**: +- Scheduled event: Market open (9:30 AM ET) +- Historical data: Traffic spikes 5x (2K β†’ 10K req/s) +- Current capacity: 5 replicas Γ— 2K req/s = 10K req/s (at limit!) +- Risk: Latency spike, potential errors + +**Analysis**: +- Need capacity for 10K req/s with headroom +- Target: 15K req/s capacity (50% headroom) +- Required replicas: 15K Γ· 2K = 7.5 β†’ 8 replicas + +**Decision**: Pre-scale 30 minutes before event +- Scale up to 8 replicas at 9:00 AM +- Monitor during event (9:30 AM - 10:00 AM) +- Scale down to 5 replicas at 11:00 AM (if traffic normalizes) + +**Action**: +```bash +# Pre-scale at 9:00 AM +kubectl scale deployment trading-service --replicas=8 -n foxhunt + +# Monitor during event +watch kubectl top pods -n foxhunt + +# Scale down at 11:00 AM (if safe) +kubectl scale deployment trading-service --replicas=5 -n foxhunt +``` + +--- + +## Cost vs Performance Tradeoffs + +### Cost Models + +**Instance Cost** (AWS t3.medium, us-east-1): +| Pricing Model | $/hour | $/month | Savings | +|---------------|--------|---------|---------| +| On-Demand | $0.0416 | $30.37 | 0% | +| Reserved (1yr) | $0.025 | $18.25 | 40% | +| Reserved (3yr) | $0.016 | $11.68 | 62% | +| Spot (avg) | $0.0125 | $9.11 | 70% | + +**Scaling Cost Example** (Trading Service): +``` +Base: 5 replicas Γ— $30.37/mo = $151.85/mo (on-demand) +Peak: 10 replicas Γ— $30.37/mo = $303.70/mo (on-demand) + +Strategy 1: All On-Demand +Cost: $303.70/mo (worst case) + +Strategy 2: Base Reserved + Peak On-Demand +Base: 5 replicas Γ— $18.25/mo = $91.25/mo (reserved) +Peak: 5 replicas Γ— $30.37/mo Γ— 30% uptime = $45.56/mo (on-demand) +Total: $136.81/mo +Savings: $166.89/mo (55% vs all on-demand) + +Strategy 3: Base Reserved + Peak Spot +Base: 5 replicas Γ— $18.25/mo = $91.25/mo (reserved) +Peak: 5 replicas Γ— $9.11/mo Γ— 30% uptime = $13.67/mo (spot) +Total: $104.92/mo +Savings: $198.78/mo (65% vs all on-demand) +``` + +### Performance Targets vs Cost + +| Target | Replicas | Cost/Month | Latency (P99) | Throughput | +|--------|----------|------------|---------------|------------| +| Minimum (HA) | 3 | $91.11 | 10-20ms | 6K req/s | +| Balanced | 5 | $151.85 | 5-10ms | 10K req/s | +| High Performance | 10 | $303.70 | 2-5ms | 20K req/s | +| Peak Capacity | 20 | $607.40 | < 2ms | 40K req/s | + +**Recommendation**: Use "Balanced" (5 replicas) as baseline, scale to 10-20 for peak traffic. + +### Cost Optimization Strategies + +#### 1. Right-Size Instances + +**Before**: +``` +Instance: t3.large (2 vCPU, 8GB RAM) +Utilization: CPU 35%, Memory 40% +Cost: $60.74/mo +``` + +**After**: +``` +Instance: t3.medium (2 vCPU, 4GB RAM) +Utilization: CPU 70%, Memory 80% +Cost: $30.37/mo +Savings: $30.37/mo (50%) +``` + +#### 2. Use Reserved Instances for Base Load + +**Before** (all on-demand): +``` +10 instances Γ— $30.37/mo = $303.70/mo +``` + +**After** (5 reserved + 5 on-demand): +``` +5 reserved Γ— $18.25/mo = $91.25/mo +5 on-demand Γ— $30.37/mo Γ— 30% uptime = $45.56/mo +Total: $136.81/mo +Savings: $166.89/mo (55%) +``` + +#### 3. Use Spot Instances for Non-Critical Workloads + +**Backtesting Workers** (spot instances): +``` +Before: 10 workers Γ— $60.74/mo (c6i.xlarge on-demand) = $607.40/mo +After: 10 workers Γ— $18.22/mo (c6i.xlarge spot, 70% discount) = $182.20/mo +Savings: $425.20/mo (70%) +``` + +**Caution**: Only use spot for fault-tolerant workloads (backtesting, ML training). Never for trading service! + +#### 4. Auto-Scale During Off-Peak Hours + +**Peak Hours** (9:00 AM - 4:00 PM ET, weekdays): +``` +Replicas: 10 (high traffic) +Cost: 10 Γ— $30.37/mo Γ— 35% time = $106.30/mo +``` + +**Off-Peak Hours** (nights, weekends): +``` +Replicas: 3 (minimal traffic) +Cost: 3 Γ— $30.37/mo Γ— 65% time = $59.22/mo +``` + +**Total Cost**: $165.52/mo (vs $303.70/mo always 10 replicas) +**Savings**: $138.18/mo (45%) + +--- + +## Scaling Procedures + +### Manual Scaling (Kubernetes) + +#### Scale Up + +```bash +# 1. Check current replica count +kubectl get deployment trading-service -n foxhunt + +# 2. Scale up +kubectl scale deployment trading-service --replicas=10 -n foxhunt + +# 3. Verify new pods are running +kubectl get pods -n foxhunt -l app=trading-service + +# 4. Wait for pods to be ready (STATUS: Running, READY: 1/1) +kubectl wait --for=condition=ready pod -l app=trading-service -n foxhunt --timeout=5m + +# 5. Verify load is distributed +kubectl top pods -n foxhunt -l app=trading-service +``` + +#### Scale Down + +```bash +# 1. Verify load is low (CPU < 50%, latency normal) +kubectl top pods -n foxhunt -l app=trading-service + +# 2. Drain pods gracefully (one at a time) +kubectl drain --ignore-daemonsets --delete-emptydir-data + +# 3. Scale down +kubectl scale deployment trading-service --replicas=5 -n foxhunt + +# 4. Verify no errors or latency spikes +# (Monitor Prometheus/Grafana for 5 minutes) +``` + +### Automated Scaling (HPA) + +#### Enable HPA + +```bash +# 1. Apply HPA manifest +kubectl apply -f config/k8s/hpa.yaml + +# 2. Verify HPA is active +kubectl get hpa -n foxhunt + +# 3. Check HPA status +kubectl describe hpa trading-service-hpa -n foxhunt + +# Expected output: +# Current replicas: 5 +# Desired replicas: 5 +# Metrics: CPU 65% (target: 70%) +``` + +#### Modify HPA Thresholds + +```bash +# 1. Edit HPA +kubectl edit hpa trading-service-hpa -n foxhunt + +# 2. Change CPU threshold (e.g., 70% β†’ 60%) +# metrics: +# - type: Resource +# resource: +# name: cpu +# target: +# type: Utilization +# averageUtilization: 60 # Changed from 70 + +# 3. Save and exit (HPA updates immediately) + +# 4. Verify change +kubectl describe hpa trading-service-hpa -n foxhunt +``` + +#### Disable HPA (Emergency) + +```bash +# 1. Delete HPA (keeps current replica count) +kubectl delete hpa trading-service-hpa -n foxhunt + +# 2. Verify HPA is gone +kubectl get hpa -n foxhunt + +# 3. Deployment is now manually managed +kubectl scale deployment trading-service --replicas= -n foxhunt +``` + +### Database Scaling + +#### Add Read Replica + +```bash +# 1. Create read replica (PostgreSQL) +# (Cloud provider specific: AWS RDS, GCP Cloud SQL, etc.) + +# Example (Kubernetes StatefulSet): +kubectl scale statefulset postgres-replica --replicas=2 -n foxhunt + +# 2. Update application to use read replica for read queries +# (See "Application-Level Read/Write Splitting" in LOAD_BALANCING_SCALING.md) + +# 3. Verify replication lag +psql -h postgres-replica-1 -c "SELECT pg_last_wal_receive_lsn(), pg_last_wal_replay_lsn();" +``` + +#### Scale Redis Cluster + +```bash +# 1. Add Redis node +redis-cli --cluster add-node new-node:6379 existing-node:6379 + +# 2. Rebalance cluster (distribute keys) +redis-cli --cluster rebalance existing-node:6379 + +# 3. Verify cluster status +redis-cli --cluster check existing-node:6379 +``` + +--- + +## Troubleshooting + +### Issue: HPA Not Scaling Up + +**Symptoms**: +- CPU > 80% but HPA not scaling +- Desired replicas = Current replicas (no change) + +**Possible Causes**: +1. **Stabilization window**: HPA waiting before scaling +2. **Resource quota exceeded**: Namespace/cluster out of resources +3. **Node capacity**: No nodes available for new pods + +**Diagnosis**: +```bash +# 1. Check HPA status +kubectl describe hpa trading-service-hpa -n foxhunt +# Look for: "ScalingLimited" or "BackoffBoth" events + +# 2. Check resource quotas +kubectl describe resourcequota -n foxhunt + +# 3. Check node capacity +kubectl describe nodes | grep -A 5 "Allocated resources" +``` + +**Solutions**: +- Wait for stabilization window to pass (30-60s) +- Increase resource quotas (if insufficient) +- Add nodes to cluster (if no capacity) + +### Issue: Pods Evicted After Scale-Up + +**Symptoms**: +- New pods created, then immediately evicted +- Pod status: "Evicted" + +**Possible Causes**: +1. **Resource pressure**: Node out of memory/disk +2. **Resource limits too high**: Pod requests exceed node capacity + +**Diagnosis**: +```bash +# 1. Check pod status +kubectl describe pod -n foxhunt +# Look for: "Reason: Evicted" + +# 2. Check node resources +kubectl top nodes + +# 3. Check pod resource requests +kubectl get pod -n foxhunt -o yaml | grep -A 10 resources +``` + +**Solutions**: +- Reduce resource requests/limits (if too high) +- Add nodes with more resources +- Delete unused pods to free resources + +### Issue: High Latency Despite Scaling Up + +**Symptoms**: +- Replicas increased (5 β†’ 10) but P99 latency still high (> 100ms) +- CPU/memory usage normal + +**Possible Causes**: +1. **Database bottleneck**: Queries slow, connection pool exhausted +2. **Network bottleneck**: Load balancer misconfigured +3. **External API bottleneck**: Dependency slow + +**Diagnosis**: +```bash +# 1. Check database query performance +psql -h postgres -c "SELECT query, mean_exec_time FROM pg_stat_statements ORDER BY mean_exec_time DESC LIMIT 10;" + +# 2. Check database connection pool +psql -h postgres -c "SELECT count(*) FROM pg_stat_activity;" + +# 3. Check external API latency +# (Prometheus query) +histogram_quantile(0.99, rate(http_request_duration_seconds_bucket{job="external-api"}[5m])) +``` + +**Solutions**: +- Optimize slow database queries (add indexes, rewrite) +- Add database read replicas +- Add PgBouncer connection pooler +- Implement caching for external API calls + +--- + +## Emergency Response + +### Incident: Service Completely Down + +**Severity**: P0 (Critical) +**Response Time**: Immediate + +**Symptoms**: +- All pods in CrashLoopBackOff +- Service unreachable (health check failing) +- Error rate: 100% + +**Immediate Actions** (within 5 minutes): + +1. **Check pod status**: + ```bash + kubectl get pods -n foxhunt -l app=trading-service + kubectl logs -n foxhunt --tail=100 + ``` + +2. **Rollback to last working version**: + ```bash + kubectl rollout undo deployment trading-service -n foxhunt + kubectl rollout status deployment trading-service -n foxhunt + ``` + +3. **Scale up to compensate**: + ```bash + kubectl scale deployment trading-service --replicas=10 -n foxhunt + ``` + +4. **Notify stakeholders**: + - Alert: "Trading Service down, rollback in progress" + - ETA: 5 minutes to restore service + +**Root Cause Analysis** (after service restored): +- Check deployment changes (git log, CI/CD) +- Check resource limits (CPU, memory, disk) +- Check external dependencies (database, Redis, external APIs) + +### Incident: Database Connection Exhaustion + +**Severity**: P1 (High) +**Response Time**: < 15 minutes + +**Symptoms**: +- Errors: "FATAL: remaining connection slots are reserved" +- Services unable to connect to database +- Error rate: 50-100% + +**Immediate Actions**: + +1. **Kill idle connections**: + ```sql + SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE state = 'idle' AND state_change < now() - interval '5 minutes'; + ``` + +2. **Scale down services temporarily**: + ```bash + # Reduce replicas to free connections + kubectl scale deployment trading-service --replicas=3 -n foxhunt + kubectl scale deployment backtesting-service --replicas=1 -n foxhunt + ``` + +3. **Deploy PgBouncer** (if not already): + ```bash + kubectl apply -f pgbouncer-deployment.yaml + ``` + +4. **Update connection strings** (rolling update): + ```bash + kubectl set env deployment/trading-service DATABASE_URL=postgresql://foxhunt:password@pgbouncer:6432/foxhunt + ``` + +### Incident: Traffic Spike (DDoS or Viral Event) + +**Severity**: P1 (High) +**Response Time**: < 10 minutes + +**Symptoms**: +- Request rate 10x normal (100K req/s vs 10K req/s) +- P99 latency > 1s +- Error rate > 5% + +**Immediate Actions**: + +1. **Enable aggressive rate limiting**: + ```nginx + # nginx config + limit_req_zone $binary_remote_addr zone=ddos:10m rate=10r/s; + limit_req zone=ddos burst=20 nodelay; + ``` + +2. **Scale up to max capacity**: + ```bash + kubectl scale deployment api-gateway --replicas=20 -n foxhunt + kubectl scale deployment trading-service --replicas=20 -n foxhunt + ``` + +3. **Enable DDoS protection** (if available): + - Cloudflare: Enable "I'm Under Attack" mode + - AWS: Enable AWS Shield Advanced + - Manual: Block suspicious IPs + +4. **Shed non-critical traffic**: + ```nginx + # Reject non-authenticated requests + location /api { + if ($http_authorization = "") { + return 503 "Service temporarily unavailable"; + } + } + ``` + +**Post-Incident**: +- Analyze traffic patterns (legitimate vs attack) +- Implement permanent DDoS protection +- Review capacity planning + +--- + +## Quick Reference + +### Common Commands + +```bash +# View current replica count +kubectl get deployment -n foxhunt + +# Scale deployment +kubectl scale deployment --replicas= -n foxhunt + +# View HPA status +kubectl get hpa -n foxhunt +kubectl describe hpa -n foxhunt + +# View pod resource usage +kubectl top pods -n foxhunt + +# View scaling events +kubectl get events -n foxhunt --field-selector involvedObject.kind=HorizontalPodAutoscaler + +# View pod logs +kubectl logs -n foxhunt --tail=100 + +# Rollback deployment +kubectl rollout undo deployment -n foxhunt +``` + +### Metrics Cheat Sheet + +| Metric | Good | Warning | Critical | +|--------|------|---------|----------| +| CPU | < 70% | 70-85% | > 85% | +| Memory | < 75% | 75-90% | > 90% | +| P99 Latency | < 10ms | 10-50ms | > 50ms | +| Error Rate | < 0.1% | 0.1-0.5% | > 0.5% | +| Replica Count | >= minReplicas | < minReplicas | 0 | + +--- + +## Additional Resources + +- **Load Balancing**: See `LOAD_BALANCING_SCALING.md` +- **Performance Benchmarks**: See `PERFORMANCE_BENCHMARKS.md` +- **Monitoring Dashboards**: Grafana (http://localhost:3000) +- **Prometheus Alerts**: Alertmanager (http://localhost:9093) + +--- + +**Last Updated**: 2025-10-07 +**Version**: 1.0 +**Owner**: Platform Engineering Team diff --git a/config/haproxy-lb.cfg b/config/haproxy-lb.cfg new file mode 100644 index 000000000..c01f1cbd8 --- /dev/null +++ b/config/haproxy-lb.cfg @@ -0,0 +1,365 @@ +# HAProxy Load Balancer Configuration for Foxhunt HFT Trading System +# Version: 1.0 +# Last Updated: 2025-10-07 +# +# This configuration provides: +# - Layer 7 (L7) gRPC load balancing +# - TLS termination at load balancer +# - Advanced health checks (gRPC Health Checking Protocol) +# - Rate limiting (via stick tables) +# - High availability with stats page +# - Connection limits + +# ============================================================================ +# GLOBAL SETTINGS +# ============================================================================ + +global + # Process management + daemon + maxconn 4096 # Maximum concurrent connections + + # Logging + log /dev/log local0 + log /dev/log local1 notice + + # User/group for HAProxy process + user haproxy + group haproxy + + # Security + chroot /var/lib/haproxy + pidfile /var/run/haproxy.pid + + # SSL/TLS settings + ssl-default-bind-ciphers ECDHE+AESGCM:ECDHE+CHACHA20:!aNULL:!MD5:!DSS + ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets + ssl-default-server-ciphers ECDHE+AESGCM:ECDHE+CHACHA20:!aNULL:!MD5:!DSS + ssl-default-server-options ssl-min-ver TLSv1.2 no-tls-tickets + + # Performance tuning + tune.ssl.default-dh-param 2048 + tune.h2.initial-window-size 65536 + tune.h2.max-concurrent-streams 128 + +# ============================================================================ +# DEFAULTS +# ============================================================================ + +defaults + # Mode (tcp for raw TCP, http for HTTP/gRPC) + mode http + + # Logging + log global + option httplog + option dontlognull + + # Timeouts + timeout connect 10s # Time to establish connection to backend + timeout client 300s # Client inactivity timeout (5 minutes for long-running gRPC) + timeout server 300s # Server inactivity timeout + timeout http-request 10s + timeout http-keep-alive 10s + + # Health checks + timeout check 5s + + # Error handling + retries 3 + option redispatch + + # HTTP/2 support (required for gRPC) + option http-use-htx # Enable HTTP/2 + +# ============================================================================ +# STATS PAGE (Monitoring) +# ============================================================================ + +listen stats + bind *:8404 + mode http + stats enable + stats uri /stats + stats refresh 10s + stats show-legends + stats show-node + + # Authentication (change username/password in production) + stats auth admin:foxhunt123 + + # Additional stats options + stats admin if TRUE # Enable admin commands (drain, disable servers) + +# ============================================================================ +# FRONTEND (External Traffic) +# ============================================================================ + +# HTTP Frontend (redirect to HTTPS) +frontend http_frontend + bind *:80 + mode http + + # Redirect all HTTP to HTTPS + http-request redirect scheme https code 301 + +# HTTPS Frontend (TLS termination + gRPC load balancing) +frontend https_frontend + # Bind to port 443 with TLS and HTTP/2 (ALPN: h2) + bind *:443 ssl crt /etc/haproxy/certs/foxhunt.pem alpn h2,http/1.1 + + mode http + option httplog + + # Connection limits (DDoS protection) + maxconn 2000 + + # ======================================================================== + # ACCESS CONTROL LISTS (ACLs) + # ======================================================================== + + # Detect gRPC protocol (Content-Type: application/grpc) + acl is_grpc hdr(content-type) -m beg application/grpc + + # Route based on gRPC service path + acl is_api_gateway path_beg /foxhunt.api_gateway + acl is_trading_service path_beg /foxhunt.trading_service + acl is_backtesting_service path_beg /foxhunt.backtesting_service + acl is_ml_training_service path_beg /foxhunt.ml_training_service + + # Internal IP ranges (for backend services) + acl is_internal_ip src 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 + + # Health check endpoint + acl is_health_check path /health + + # ======================================================================== + # RATE LIMITING (Stick Tables) + # ======================================================================== + + # Track request rate per client IP + stick-table type ip size 100k expire 30s store http_req_rate(10s) + + # Track connection rate per client IP + http-request track-sc0 src + + # Rate limit: 1000 requests per 10 seconds per IP + http-request deny deny_status 429 if { sc_http_req_rate(0) gt 1000 } + + # Connection limit: 100 concurrent connections per IP + http-request deny deny_status 429 if { src_conn_cur ge 100 } + + # ======================================================================== + # REQUEST HEADERS + # ======================================================================== + + # Add X-Forwarded-* headers (for backend logging) + http-request set-header X-Real-IP %[src] + http-request set-header X-Forwarded-For %[src] + http-request set-header X-Forwarded-Proto https + + # ======================================================================== + # ROUTING + # ======================================================================== + + # Health check (return 200 OK) + use_backend health_backend if is_health_check + + # Route gRPC traffic to appropriate backend + use_backend api_gateway_backend if is_grpc is_api_gateway + use_backend trading_service_backend if is_grpc is_trading_service is_internal_ip + use_backend backtesting_service_backend if is_grpc is_backtesting_service is_internal_ip + use_backend ml_training_service_backend if is_grpc is_ml_training_service is_internal_ip + + # Default: Return 404 for unknown routes + default_backend not_found_backend + +# ============================================================================ +# BACKENDS (Service Clusters) +# ============================================================================ + +# API Gateway Backend (primary entry point) +backend api_gateway_backend + mode http + balance leastconn # Route to backend with fewest connections (best for gRPC) + + # HTTP/2 support + option http-use-htx + + # Health check using gRPC Health Checking Protocol + option httpchk + http-check send meth GET uri /grpc.health.v1.Health/Check ver HTTP/2 + http-check expect status 200 + + # Backend servers + # Format: server : [options] + server api-gateway-1 api-gateway-1:50051 check inter 5s rise 2 fall 3 maxconn 1000 + server api-gateway-2 api-gateway-2:50051 check inter 5s rise 2 fall 3 maxconn 1000 + server api-gateway-3 api-gateway-3:50051 check inter 5s rise 2 fall 3 maxconn 1000 + + # Connection reuse (important for gRPC performance) + http-reuse always + +# Trading Service Backend +backend trading_service_backend + mode http + balance leastconn + + option http-use-htx + + # Health check + option httpchk + http-check send meth GET uri /grpc.health.v1.Health/Check ver HTTP/2 + http-check expect status 200 + + # Backend servers + server trading-service-1 trading-service-1:50052 check inter 5s rise 2 fall 3 maxconn 1000 + server trading-service-2 trading-service-2:50052 check inter 5s rise 2 fall 3 maxconn 1000 + server trading-service-3 trading-service-3:50052 check inter 5s rise 2 fall 3 maxconn 1000 + + http-reuse always + +# Backtesting Service Backend (worker pool pattern) +backend backtesting_service_backend + mode http + balance leastconn + + option http-use-htx + + # Health check + option httpchk + http-check send meth GET uri /grpc.health.v1.Health/Check ver HTTP/2 + http-check expect status 200 + + # Backend servers (fewer instances, CPU-intensive workload) + server backtesting-service-1 backtesting-service-1:50053 check inter 5s rise 2 fall 3 maxconn 500 + server backtesting-service-2 backtesting-service-2:50053 check inter 5s rise 2 fall 3 maxconn 500 + + http-reuse always + +# ML Training Service Backend (GPU instances) +backend ml_training_service_backend + mode http + balance leastconn + + option http-use-htx + + # Health check + option httpchk + http-check send meth GET uri /grpc.health.v1.Health/Check ver HTTP/2 + http-check expect status 200 + + # Backend servers (GPU instances, fewer replicas) + server ml-training-service-1 ml-training-service-1:50054 check inter 5s rise 2 fall 3 maxconn 200 + server ml-training-service-2 ml-training-service-2:50054 check inter 5s rise 2 fall 3 maxconn 200 + + http-reuse always + +# Health Check Backend (returns 200 OK) +backend health_backend + mode http + http-request return status 200 content-type "text/plain" string "healthy\n" + +# Not Found Backend (returns 404) +backend not_found_backend + mode http + http-request return status 404 content-type "text/plain" string "not found\n" + +# ============================================================================ +# MONITORING & OBSERVABILITY +# ============================================================================ + +# HAProxy Stats (available at http://localhost:8404/stats) +# Metrics include: +# - Request rate, error rate, latency +# - Active connections per backend +# - Health check status +# - Session rate + +# Prometheus Exporter: +# Use haproxy_exporter: https://github.com/prometheus/haproxy_exporter +# docker run -d -p 9101:9101 prom/haproxy-exporter:latest \ +# --haproxy.scrape-uri="http://admin:foxhunt123@localhost:8404/stats;csv" + +# Example Prometheus scrape config: +# scrape_configs: +# - job_name: 'haproxy' +# static_configs: +# - targets: ['haproxy:9101'] + +# ============================================================================ +# NOTES +# ============================================================================ + +# 1. SSL Certificate: +# - Combine certificate and key into single PEM file: +# cat foxhunt.crt foxhunt.key > /etc/haproxy/certs/foxhunt.pem +# - Ensure correct permissions: chmod 600 /etc/haproxy/certs/foxhunt.pem +# +# 2. Health Checks: +# - Uses gRPC Health Checking Protocol (standard) +# - Check interval: 5 seconds +# - Rise: 2 successful checks = healthy +# - Fall: 3 failed checks = unhealthy +# - Backends must implement grpc.health.v1.Health service +# +# 3. Rate Limiting: +# - Stick tables track request rate per IP +# - Adjust limits based on traffic patterns +# - Monitor 429 error rate to avoid over-throttling +# +# 4. Load Balancing Algorithm: +# - leastconn: Best for gRPC (long-lived connections) +# - Alternative: roundrobin (simpler, less accurate) +# - Alternative: source (session affinity by IP) +# +# 5. Connection Reuse: +# - http-reuse always: Reuse connections to backends (critical for gRPC) +# - Reduces latency and connection overhead +# +# 6. Logging: +# - Logs sent to syslog (/dev/log) +# - Configure rsyslog to route HAProxy logs to file +# - Example: /var/log/haproxy.log + +# ============================================================================ +# PRODUCTION CHECKLIST +# ============================================================================ + +# [ ] Replace SSL certificate path (/etc/haproxy/certs/foxhunt.pem) +# [ ] Change stats page credentials (admin:foxhunt123) +# [ ] Configure DNS for foxhunt.trading +# [ ] Set up log rotation (logrotate) +# [ ] Enable Prometheus exporter (haproxy_exporter) +# [ ] Configure alerts (high error rate, backend down) +# [ ] Test rate limiting with load testing tools +# [ ] Configure firewall rules (allow 80, 443, 8404; deny others) +# [ ] Document backend server IP addresses +# [ ] Test failover (kill backend, verify traffic reroutes) +# [ ] Set up automated certificate renewal (certbot cron) + +# ============================================================================ +# HAPROXY MANAGEMENT COMMANDS +# ============================================================================ + +# Start HAProxy: +# haproxy -f /etc/haproxy/haproxy.cfg +# +# Check configuration: +# haproxy -c -f /etc/haproxy/haproxy.cfg +# +# Reload configuration (zero-downtime): +# haproxy -f /etc/haproxy/haproxy.cfg -sf $(cat /var/run/haproxy.pid) +# +# View stats: +# http://localhost:8404/stats +# +# Admin commands (via stats page): +# - Drain server: Set server to "drain" mode (no new connections) +# - Disable server: Take server out of rotation +# - Enable server: Put server back into rotation +# +# Socket commands (requires stats socket): +# echo "show stat" | socat stdio /var/run/haproxy.sock +# echo "show servers state" | socat stdio /var/run/haproxy.sock +# echo "disable server api_gateway_backend/api-gateway-1" | socat stdio /var/run/haproxy.sock diff --git a/config/k8s/hpa.yaml b/config/k8s/hpa.yaml new file mode 100644 index 000000000..45c5d659b --- /dev/null +++ b/config/k8s/hpa.yaml @@ -0,0 +1,464 @@ +# Kubernetes Horizontal Pod Autoscaler (HPA) Manifests +# Version: 1.0 +# Last Updated: 2025-10-07 +# +# These HPA configurations provide: +# - CPU-based autoscaling for all services +# - Custom metrics for advanced scaling (request rate, queue depth) +# - Configurable scale-up/scale-down behavior +# - Min/max replica counts for cost optimization + +--- +# ============================================================================ +# API GATEWAY HPA (Latency-Sensitive, Aggressive Scaling) +# ============================================================================ + +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: api-gateway-hpa + namespace: foxhunt + labels: + app: api-gateway + component: gateway +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: api-gateway + + # Replica limits + minReplicas: 3 # Always maintain 3 for HA (survive 1 node failure + 1 rolling update) + maxReplicas: 20 # Max capacity: 20 instances Γ— 2K req/s = 40K req/s + + # Scaling metrics + metrics: + # CPU-based scaling (primary metric) + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 # Scale when CPU > 70% + + # Memory-based scaling (secondary metric) + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 # Scale when memory > 80% + + # Custom metric: Request rate (requires Prometheus adapter) + - type: Pods + pods: + metric: + name: grpc_requests_per_second + target: + type: AverageValue + averageValue: "2000" # Scale when > 2000 req/s per pod + + # Scaling behavior (aggressive scale-up, slow scale-down) + behavior: + scaleUp: + stabilizationWindowSeconds: 30 # Wait 30s before scaling up + policies: + - type: Percent + value: 100 # Double replicas + periodSeconds: 15 + - type: Pods + value: 4 # Or add 4 pods + periodSeconds: 15 + selectPolicy: Max # Use whichever scales faster + + scaleDown: + stabilizationWindowSeconds: 300 # Wait 5 minutes before scaling down + policies: + - type: Percent + value: 25 # Remove 25% of replicas + periodSeconds: 60 + - type: Pods + value: 1 # Or remove 1 pod + periodSeconds: 60 + selectPolicy: Min # Use whichever scales slower + +--- +# ============================================================================ +# TRADING SERVICE HPA (Latency-Sensitive, Aggressive Scaling) +# ============================================================================ + +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: trading-service-hpa + namespace: foxhunt + labels: + app: trading-service + component: backend +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: trading-service + + # Replica limits + minReplicas: 3 # HA requirement + maxReplicas: 20 # Based on performance testing: 50K orders/sec Γ· 2.5K orders/sec per pod + + # Scaling metrics + metrics: + # CPU-based scaling + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + + # Memory-based scaling + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 + + # Custom metric: Order submission rate + - type: Pods + pods: + metric: + name: order_submissions_per_second + target: + type: AverageValue + averageValue: "2500" # Scale when > 2500 orders/s per pod + + # Scaling behavior (aggressive scale-up) + behavior: + scaleUp: + stabilizationWindowSeconds: 30 + policies: + - type: Percent + value: 100 + periodSeconds: 15 + - type: Pods + value: 4 + periodSeconds: 15 + selectPolicy: Max + + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 25 + periodSeconds: 60 + - type: Pods + value: 1 + periodSeconds: 60 + selectPolicy: Min + +--- +# ============================================================================ +# BACKTESTING SERVICE HPA (Job Queue Pattern, Conservative Scaling) +# ============================================================================ + +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: backtesting-service-hpa + namespace: foxhunt + labels: + app: backtesting-service + component: backend +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: backtesting-service + + # Replica limits + minReplicas: 2 # Minimum workers for job queue + maxReplicas: 10 # Rarely need more than 10 concurrent backtests + + # Scaling metrics + metrics: + # CPU-based scaling (primary for CPU-intensive backtests) + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 75 # Higher threshold for batch workload + + # Memory-based scaling + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 + + # Custom metric: Redis job queue depth + - type: Pods + pods: + metric: + name: redis_queue_length_backtesting + target: + type: AverageValue + averageValue: "5" # Scale when > 5 jobs per worker + + # Scaling behavior (conservative scale-up, slower scale-down) + behavior: + scaleUp: + stabilizationWindowSeconds: 60 # Wait 60s before scaling up + policies: + - type: Percent + value: 50 # Add 50% more replicas + periodSeconds: 60 + - type: Pods + value: 2 # Or add 2 pods + periodSeconds: 60 + selectPolicy: Min # Use whichever scales slower + + scaleDown: + stabilizationWindowSeconds: 600 # Wait 10 minutes before scaling down + policies: + - type: Percent + value: 25 + periodSeconds: 120 + - type: Pods + value: 1 + periodSeconds: 120 + selectPolicy: Min + +--- +# ============================================================================ +# ML TRAINING SERVICE HPA (MANUAL SCALING RECOMMENDED) +# ============================================================================ + +# NOTE: ML Training Service uses GPU resources, which are expensive and slow to provision. +# Recommendation: Use manual scaling (kubectl scale) or scheduled scaling, NOT auto-scaling. +# +# If auto-scaling is required, use job queue pattern with custom metrics: + +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: ml-training-service-hpa + namespace: foxhunt + labels: + app: ml-training-service + component: backend +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: ml-training-service + + # Replica limits (GPU resources expensive) + minReplicas: 1 # Single instance (GPU idle time acceptable) + maxReplicas: 3 # Limited by GPU cluster size + + # Scaling metrics + metrics: + # GPU utilization (requires NVIDIA DCGM exporter) + - type: Pods + pods: + metric: + name: dcgm_gpu_utilization + target: + type: AverageValue + averageValue: "80" # Scale when GPU > 80% utilized + + # Custom metric: Training job queue depth + - type: Pods + pods: + metric: + name: redis_queue_length_ml_training + target: + type: AverageValue + averageValue: "2" # Scale when > 2 training jobs queued + + # Scaling behavior (very conservative, GPU instances expensive) + behavior: + scaleUp: + stabilizationWindowSeconds: 300 # Wait 5 minutes before scaling up + policies: + - type: Pods + value: 1 # Add 1 pod at a time + periodSeconds: 300 + selectPolicy: Min + + scaleDown: + stabilizationWindowSeconds: 1800 # Wait 30 minutes before scaling down + policies: + - type: Pods + value: 1 + periodSeconds: 300 + selectPolicy: Min + +--- +# ============================================================================ +# CUSTOM METRICS (Prometheus Adapter Configuration) +# ============================================================================ + +# To use custom metrics (grpc_requests_per_second, redis_queue_length, etc.), +# you must install and configure Prometheus Adapter: +# +# 1. Install Prometheus Adapter: +# helm install prometheus-adapter prometheus-community/prometheus-adapter \ +# --namespace monitoring \ +# --values prometheus-adapter-values.yaml +# +# 2. Configure custom metrics (prometheus-adapter-values.yaml): + +# prometheus: +# url: http://prometheus-server.monitoring.svc +# port: 9090 +# +# rules: +# default: false +# custom: +# # gRPC request rate per pod +# - seriesQuery: 'grpc_requests_total{namespace="foxhunt"}' +# resources: +# overrides: +# namespace: {resource: "namespace"} +# pod: {resource: "pod"} +# name: +# matches: "^grpc_requests_total" +# as: "grpc_requests_per_second" +# metricsQuery: 'sum(rate(grpc_requests_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)' +# +# # Redis queue length +# - seriesQuery: 'redis_list_length{namespace="foxhunt",queue="backtesting"}' +# resources: +# overrides: +# namespace: {resource: "namespace"} +# name: +# matches: "^redis_list_length" +# as: "redis_queue_length_backtesting" +# metricsQuery: 'avg(redis_list_length{<<.LabelMatchers>>}) by (<<.GroupBy>>)' +# +# # GPU utilization (requires NVIDIA DCGM exporter) +# - seriesQuery: 'DCGM_FI_DEV_GPU_UTIL{namespace="foxhunt"}' +# resources: +# overrides: +# namespace: {resource: "namespace"} +# pod: {resource: "pod"} +# name: +# matches: "^DCGM_FI_DEV_GPU_UTIL" +# as: "dcgm_gpu_utilization" +# metricsQuery: 'avg(DCGM_FI_DEV_GPU_UTIL{<<.LabelMatchers>>}) by (<<.GroupBy>>)' + +# 3. Verify custom metrics are available: +# kubectl get --raw "/apis/custom.metrics.k8s.io/v1beta1" | jq . + +# 4. Test HPA with custom metrics: +# kubectl get hpa api-gateway-hpa -o yaml +# # Check "currentMetrics" section for custom metric values + +--- +# ============================================================================ +# TESTING & VALIDATION +# ============================================================================ + +# 1. Deploy HPA manifests: +# kubectl apply -f hpa.yaml +# +# 2. Verify HPA status: +# kubectl get hpa -n foxhunt +# kubectl describe hpa api-gateway-hpa -n foxhunt +# +# 3. Monitor scaling events: +# kubectl get events -n foxhunt --field-selector involvedObject.kind=HorizontalPodAutoscaler +# +# 4. Load test to trigger scaling: +# # Generate load (e.g., with ghz) +# ghz --insecure \ +# --proto api_gateway.proto \ +# --call foxhunt.api_gateway.ApiGateway/SubmitOrder \ +# --rps 10000 \ +# --duration 5m \ +# localhost:50051 +# +# # Watch HPA scale up +# watch kubectl get hpa -n foxhunt +# +# 5. Verify scale-down after load stops: +# # Wait 5-10 minutes after load test ends +# kubectl get hpa api-gateway-hpa -n foxhunt +# # Should scale down to minReplicas + +--- +# ============================================================================ +# TROUBLESHOOTING +# ============================================================================ + +# Issue: HPA shows "unknown" for metrics +# Solution: Ensure metrics-server is installed and running +# kubectl get deployment metrics-server -n kube-system +# kubectl logs -n kube-system deployment/metrics-server +# +# Issue: Custom metrics not available +# Solution: Verify Prometheus Adapter configuration +# kubectl get --raw "/apis/custom.metrics.k8s.io/v1beta1" | jq . +# kubectl logs -n monitoring deployment/prometheus-adapter +# +# Issue: HPA not scaling up despite high CPU +# Solution: Check stabilizationWindowSeconds (may be waiting) +# kubectl describe hpa -n foxhunt +# # Look for "ScalingLimited" or "BackoffBoth" events +# +# Issue: HPA scaling too aggressively (flapping) +# Solution: Increase stabilizationWindowSeconds or reduce scale-up policies +# +# Issue: Pods evicted due to resource pressure +# Solution: Increase resource requests/limits in Deployment spec +# kubectl describe pod -n foxhunt +# # Look for "Evicted" status + +--- +# ============================================================================ +# PRODUCTION BEST PRACTICES +# ============================================================================ + +# 1. Monitor HPA metrics: +# - Current replicas vs desired replicas +# - Scaling events (up/down) +# - Metric values (CPU, memory, custom) +# +# 2. Alert on scaling issues: +# - HPA unable to scale (resource quotas exceeded) +# - HPA scaling too frequently (flapping) +# - Pods in CrashLoopBackOff after scale-up +# +# 3. Test scaling under load: +# - Run load tests regularly (weekly/monthly) +# - Verify scale-up happens within SLA (< 2 minutes) +# - Verify scale-down doesn't cause service disruption +# +# 4. Cost optimization: +# - Use Reserved Instances for minReplicas (50% savings) +# - Use Spot Instances for auto-scaled replicas (70% savings, non-critical workloads) +# - Monitor auto-scaling costs (CloudWatch/Prometheus) +# +# 5. Resource limits: +# - Set resource requests = expected usage +# - Set resource limits = 2x requests (headroom for bursts) +# - Monitor actual usage and adjust + +--- +# ============================================================================ +# PRODUCTION CHECKLIST +# ============================================================================ + +# [ ] Install metrics-server (for CPU/memory metrics) +# [ ] Install Prometheus Adapter (for custom metrics) +# [ ] Configure custom metrics (request rate, queue depth, GPU util) +# [ ] Deploy HPA manifests (kubectl apply -f hpa.yaml) +# [ ] Verify HPA status (kubectl get hpa) +# [ ] Run load test to validate scaling behavior +# [ ] Configure Grafana dashboards (HPA metrics, scaling events) +# [ ] Configure alerts (scaling issues, resource pressure) +# [ ] Document scaling thresholds and replica limits +# [ ] Review and adjust HPA configuration quarterly diff --git a/config/nginx-lb.conf b/config/nginx-lb.conf new file mode 100644 index 000000000..957bafea9 --- /dev/null +++ b/config/nginx-lb.conf @@ -0,0 +1,331 @@ +# nginx Load Balancer Configuration for Foxhunt HFT Trading System +# Version: 1.0 +# Last Updated: 2025-10-07 +# +# This configuration provides: +# - Layer 7 (L7) gRPC load balancing +# - TLS termination at load balancer +# - Health checks using gRPC Health Checking Protocol +# - Rate limiting by user tier +# - DDoS protection +# - Connection limits + +# ============================================================================ +# UPSTREAM DEFINITIONS +# ============================================================================ + +# API Gateway cluster (primary entry point for all clients) +upstream api_gateway_backend { + # Load balancing algorithm + # - least_conn: Route to backend with fewest active connections (recommended for long-lived gRPC connections) + # - round_robin: Default, simple rotation + # - ip_hash: Session affinity based on client IP (use for stateful operations) + least_conn; + + # Backend servers + # Format: server : [parameters]; + server api-gateway-1:50051 max_fails=3 fail_timeout=30s; + server api-gateway-2:50051 max_fails=3 fail_timeout=30s; + server api-gateway-3:50051 max_fails=3 fail_timeout=30s; + + # Health check (requires nginx Plus or nginx-module-health-check) + # For open-source nginx, use passive health checks (max_fails) + # For nginx Plus: + # health_check interval=5s fails=2 passes=1 uri=/grpc.health.v1.Health/Check; + + # Keepalive connections to backend + # Reuse connections instead of creating new ones (critical for gRPC performance) + keepalive 100; + keepalive_timeout 60s; +} + +# Trading Service cluster (internal, routed via API Gateway) +upstream trading_service_backend { + least_conn; + server trading-service-1:50052 max_fails=3 fail_timeout=30s; + server trading-service-2:50052 max_fails=3 fail_timeout=30s; + server trading-service-3:50052 max_fails=3 fail_timeout=30s; + keepalive 100; +} + +# Backtesting Service cluster (worker pool pattern) +upstream backtesting_service_backend { + least_conn; + server backtesting-service-1:50053 max_fails=3 fail_timeout=30s; + server backtesting-service-2:50053 max_fails=3 fail_timeout=30s; + keepalive 50; +} + +# ML Training Service cluster (GPU instances, fewer replicas) +upstream ml_training_service_backend { + least_conn; + server ml-training-service-1:50054 max_fails=3 fail_timeout=30s; + server ml-training-service-2:50054 max_fails=3 fail_timeout=30s; + keepalive 20; +} + +# ============================================================================ +# RATE LIMITING ZONES +# ============================================================================ + +# Rate limiting by user tier (extracted from JWT) +# Zone size: 10m = 10 MB = ~160,000 IP addresses + +# Free tier: 100 requests/second +limit_req_zone $jwt_user_tier zone=tier_free:10m rate=100r/s; + +# Premium tier: 500 requests/second +limit_req_zone $jwt_user_tier zone=tier_premium:10m rate=500r/s; + +# Enterprise tier: 2000 requests/second +limit_req_zone $jwt_user_tier zone=tier_enterprise:10m rate=2000r/s; + +# Internal services: No rate limit +# (Identified by internal JWT or mTLS certificate) + +# Connection limits per IP (DDoS protection) +limit_conn_zone $binary_remote_addr zone=addr:10m; + +# ============================================================================ +# MAP DEFINITIONS +# ============================================================================ + +# Extract user tier from JWT (requires lua-nginx-module or custom logic) +# For simplicity, this example assumes tier is in a custom header +# In production, extract from JWT claims +map $http_x_user_tier $rate_limit_zone { + default tier_free; + "free" tier_free; + "premium" tier_premium; + "enterprise" tier_enterprise; + "internal" ""; # No rate limit for internal services +} + +# ============================================================================ +# SERVER BLOCKS +# ============================================================================ + +# HTTP server (redirect to HTTPS) +server { + listen 80; + listen [::]:80; + server_name foxhunt.trading; + + # Redirect all HTTP traffic to HTTPS + return 301 https://$server_name$request_uri; +} + +# HTTPS server (TLS termination + gRPC load balancing) +server { + # Listen on port 443 with HTTP/2 (required for gRPC) + listen 443 ssl http2; + listen [::]:443 ssl http2; + + server_name foxhunt.trading; + + # ======================================================================== + # TLS CONFIGURATION + # ======================================================================== + + # SSL certificate (replace with your actual certificate paths) + ssl_certificate /etc/ssl/certs/foxhunt.crt; + ssl_certificate_key /etc/ssl/private/foxhunt.key; + + # TLS protocols (TLS 1.2 and 1.3 only, no SSLv3/TLS 1.0/1.1) + ssl_protocols TLSv1.2 TLSv1.3; + + # Cipher suites (prefer modern, secure ciphers) + ssl_ciphers HIGH:!aNULL:!MD5:!3DES; + ssl_prefer_server_ciphers on; + + # SSL session cache (improves performance by reusing TLS sessions) + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 10m; + + # OCSP stapling (improves TLS handshake performance) + ssl_stapling on; + ssl_stapling_verify on; + ssl_trusted_certificate /etc/ssl/certs/ca-bundle.crt; + + # HSTS (HTTP Strict Transport Security) + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + + # ======================================================================== + # GRPC CONFIGURATION + # ======================================================================== + + # gRPC-specific settings + grpc_read_timeout 300s; # 5 minutes (for long-running backtests) + grpc_send_timeout 30s; + grpc_connect_timeout 10s; + + # Buffer sizes (tune based on payload size) + client_body_buffer_size 1M; + client_max_body_size 10M; + + # HTTP/2 settings + http2_max_concurrent_streams 128; + http2_recv_timeout 300s; + + # ======================================================================== + # RATE LIMITING & DDOS PROTECTION + # ======================================================================== + + # Connection limit per IP (max 10 concurrent connections) + limit_conn addr 10; + + # Rate limiting by user tier (applied per location) + # See location blocks below + + # ======================================================================== + # LOGGING + # ======================================================================== + + # Access log (includes gRPC status) + access_log /var/log/nginx/access.log combined; + + # Error log + error_log /var/log/nginx/error.log warn; + + # ======================================================================== + # LOCATIONS (ROUTING) + # ======================================================================== + + # API Gateway (primary entry point for clients) + location /foxhunt.api_gateway { + # Apply rate limiting based on user tier + limit_req zone=$rate_limit_zone burst=20 nodelay; + + # gRPC proxy to backend + grpc_pass grpc://api_gateway_backend; + + # Pass original client IP (for logging and rate limiting) + grpc_set_header X-Real-IP $remote_addr; + grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + grpc_set_header X-Forwarded-Proto $scheme; + + # Error handling + grpc_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; + grpc_next_upstream_tries 3; + grpc_next_upstream_timeout 10s; + } + + # Trading Service (internal, typically accessed via API Gateway) + # Exposed for direct access if needed (e.g., for monitoring) + location /foxhunt.trading_service { + # Restrict to internal IPs only + allow 10.0.0.0/8; # Internal network + allow 172.16.0.0/12; # Docker network + deny all; + + grpc_pass grpc://trading_service_backend; + grpc_set_header X-Real-IP $remote_addr; + grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + # Backtesting Service (internal) + location /foxhunt.backtesting_service { + allow 10.0.0.0/8; + allow 172.16.0.0/12; + deny all; + + grpc_pass grpc://backtesting_service_backend; + grpc_set_header X-Real-IP $remote_addr; + grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + # ML Training Service (internal) + location /foxhunt.ml_training_service { + allow 10.0.0.0/8; + allow 172.16.0.0/12; + deny all; + + grpc_pass grpc://ml_training_service_backend; + grpc_set_header X-Real-IP $remote_addr; + grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + # Health check endpoint (HTTP, not gRPC) + location /health { + access_log off; + return 200 "healthy\n"; + add_header Content-Type text/plain; + } + + # Metrics endpoint (for Prometheus scraping) + location /metrics { + # Restrict to monitoring IPs only + allow 10.0.0.0/8; + deny all; + + # Stub status (requires --with-http_stub_status_module) + stub_status; + } + + # Deny all other requests + location / { + return 404; + } +} + +# ============================================================================ +# MONITORING & OBSERVABILITY +# ============================================================================ + +# Prometheus metrics exporter (requires nginx-prometheus-exporter) +# Run separately as a sidecar container: +# docker run -p 9113:9113 nginx/nginx-prometheus-exporter:latest \ +# --nginx.scrape-uri=http://localhost:8080/metrics + +# Example Prometheus scrape config: +# scrape_configs: +# - job_name: 'nginx-lb' +# static_configs: +# - targets: ['nginx-lb:9113'] + +# ============================================================================ +# NOTES +# ============================================================================ + +# 1. Replace SSL certificate paths with your actual certificates: +# - Production: Use Let's Encrypt (certbot) or corporate CA +# - Development: Self-signed certificates (not recommended for production) +# +# 2. Adjust rate limits based on your traffic patterns: +# - Monitor 429 (Too Many Requests) error rate in Prometheus +# - Increase limits if legitimate users are throttled +# +# 3. Health checks: +# - Open-source nginx: Uses passive health checks (max_fails) +# - nginx Plus: Active health checks with /grpc.health.v1.Health/Check +# - Alternative: Use external health checker (e.g., Kubernetes liveness probes) +# +# 4. Load balancing algorithms: +# - least_conn: Best for gRPC (long-lived connections) +# - round_robin: Simpler, but can lead to uneven load +# - ip_hash: Use only if session affinity is required +# +# 5. Monitoring: +# - Use nginx-prometheus-exporter for metrics +# - Monitor: request rate, error rate, latency, active connections +# - Alerts: high error rate (> 1%), high latency (P99 > 100ms) +# +# 6. Scaling: +# - Add more backend servers to upstream blocks +# - Reload nginx config: nginx -s reload (zero-downtime) +# - For Kubernetes: Use Ingress controller (nginx-ingress) + +# ============================================================================ +# PRODUCTION CHECKLIST +# ============================================================================ + +# [ ] Replace SSL certificates with production certificates +# [ ] Configure DNS for foxhunt.trading +# [ ] Set up log rotation (logrotate) +# [ ] Enable Prometheus metrics exporter +# [ ] Configure alerts (high error rate, high latency) +# [ ] Test rate limiting with load testing tools +# [ ] Configure firewall rules (allow 80, 443; deny others) +# [ ] Enable nginx status page (for debugging) +# [ ] Document backend server IP addresses +# [ ] Set up automated certificate renewal (certbot cron) diff --git a/run_smoke_tests.sh b/run_smoke_tests.sh new file mode 100755 index 000000000..4b4aa2f1e --- /dev/null +++ b/run_smoke_tests.sh @@ -0,0 +1,218 @@ +#!/bin/bash +# Foxhunt HFT System - Automated Smoke Test Suite +# Version: 1.0.0 +# Purpose: Validate complete system functionality after deployment +# +# Usage: +# ./run_smoke_tests.sh # Run all smoke tests +# ./run_smoke_tests.sh --fast # Run only critical tests +# ./run_smoke_tests.sh --verbose # Run with detailed logging +# ./run_smoke_tests.sh --category infrastructure # Run specific category + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Configuration +VERBOSE=false +FAST_MODE=false +CATEGORY="" +RUST_LOG="${RUST_LOG:-info}" + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + --verbose|-v) + VERBOSE=true + RUST_LOG="debug" + shift + ;; + --fast|-f) + FAST_MODE=true + shift + ;; + --category|-c) + CATEGORY="$2" + shift 2 + ;; + --help|-h) + echo "Foxhunt HFT System - Smoke Test Runner" + echo "" + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " --verbose, -v Enable verbose logging (RUST_LOG=debug)" + echo " --fast, -f Run only critical tests (faster execution)" + echo " --category, -c CAT Run specific category only" + echo " Categories: infrastructure, service, authentication, order_flow" + echo " --help, -h Show this help message" + echo "" + echo "Examples:" + echo " $0 # Run all smoke tests" + echo " $0 --fast # Run only critical tests" + echo " $0 --verbose # Run with debug logging" + echo " $0 --category infrastructure # Run infrastructure tests only" + exit 0 + ;; + *) + echo -e "${RED}Unknown option: $1${NC}" + echo "Use --help for usage information" + exit 1 + ;; + esac +done + +# Export environment variables +export RUST_LOG +export DATABASE_URL="${DATABASE_URL:-postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt}" +export REDIS_URL="${REDIS_URL:-redis://localhost:6379}" +export VAULT_ADDR="${VAULT_ADDR:-http://localhost:8200}" +export INFLUXDB_URL="${INFLUXDB_URL:-http://localhost:8086}" +export API_GATEWAY_URL="${API_GATEWAY_URL:-http://localhost:50051}" +export TRADING_SERVICE_URL="${TRADING_SERVICE_URL:-http://localhost:50052}" +export BACKTESTING_SERVICE_URL="${BACKTESTING_SERVICE_URL:-http://localhost:50053}" +export ML_TRAINING_SERVICE_URL="${ML_TRAINING_SERVICE_URL:-http://localhost:50054}" +export PROMETHEUS_URL="${PROMETHEUS_URL:-http://localhost:9090}" +export GRAFANA_URL="${GRAFANA_URL:-http://localhost:3000}" +export JWT_SECRET="${JWT_SECRET:-dev_secret_key_change_in_production}" + +# Print banner +echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" +echo -e "${BLUE} Foxhunt HFT System - Smoke Test Suite${NC}" +echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" +echo "" + +# Environment info +echo -e "${YELLOW}Environment Configuration:${NC}" +echo " Database: ${DATABASE_URL}" +echo " Redis: ${REDIS_URL}" +echo " Vault: ${VAULT_ADDR}" +echo " API Gateway: ${API_GATEWAY_URL}" +echo " Trading Service: ${TRADING_SERVICE_URL}" +echo " Log Level: ${RUST_LOG}" +if [ "$FAST_MODE" = true ]; then + echo -e " ${GREEN}Mode: FAST (critical tests only)${NC}" +fi +if [ "$CATEGORY" != "" ]; then + echo -e " ${GREEN}Category: ${CATEGORY}${NC}" +fi +echo "" + +# Test execution flags +TEST_FLAGS="" +if [ "$VERBOSE" = true ]; then + TEST_FLAGS="-- --nocapture --test-threads=1" +fi + +# Function to run a test category +run_test_category() { + local category=$1 + local test_name=$2 + local description=$3 + + echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo -e "${BLUE}Testing: ${description}${NC}" + echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo "" + + if cargo test --test smoke_tests ${test_name} ${TEST_FLAGS}; then + echo "" + echo -e "${GREEN}βœ… ${description} - PASSED${NC}" + return 0 + else + echo "" + echo -e "${RED}❌ ${description} - FAILED${NC}" + return 1 + fi +} + +# Track results +TOTAL_CATEGORIES=0 +PASSED_CATEGORIES=0 +FAILED_CATEGORIES=0 + +# Main test execution +echo -e "${YELLOW}πŸ” Starting smoke test execution...${NC}" +echo "" + +# Category 1: Infrastructure Health (CRITICAL) +if [ -z "$CATEGORY" ] || [ "$CATEGORY" = "infrastructure" ]; then + TOTAL_CATEGORIES=$((TOTAL_CATEGORIES + 1)) + if run_test_category "infrastructure" "infrastructure_health" "Infrastructure Health"; then + PASSED_CATEGORIES=$((PASSED_CATEGORIES + 1)) + else + FAILED_CATEGORIES=$((FAILED_CATEGORIES + 1)) + fi + echo "" +fi + +# Category 2: Service Health (CRITICAL) +if [ -z "$CATEGORY" ] || [ "$CATEGORY" = "service" ]; then + TOTAL_CATEGORIES=$((TOTAL_CATEGORIES + 1)) + if run_test_category "service" "service_health" "Service Health"; then + PASSED_CATEGORIES=$((PASSED_CATEGORIES + 1)) + else + FAILED_CATEGORIES=$((FAILED_CATEGORIES + 1)) + echo -e "${YELLOW}⚠️ Note: Some services may not be available due to configuration issues${NC}" + fi + echo "" +fi + +# Category 3: Authentication Flow (skip in fast mode) +if [ "$FAST_MODE" = false ] && ([ -z "$CATEGORY" ] || [ "$CATEGORY" = "authentication" ]); then + TOTAL_CATEGORIES=$((TOTAL_CATEGORIES + 1)) + if run_test_category "authentication" "authentication_flow" "Authentication Flow"; then + PASSED_CATEGORIES=$((PASSED_CATEGORIES + 1)) + else + FAILED_CATEGORIES=$((FAILED_CATEGORIES + 1)) + fi + echo "" +fi + +# Category 4: Basic Order Flow (skip in fast mode) +if [ "$FAST_MODE" = false ] && ([ -z "$CATEGORY" ] || [ "$CATEGORY" = "order_flow" ]); then + TOTAL_CATEGORIES=$((TOTAL_CATEGORIES + 1)) + if run_test_category "order_flow" "basic_order_flow" "Basic Order Flow"; then + PASSED_CATEGORIES=$((PASSED_CATEGORIES + 1)) + else + FAILED_CATEGORIES=$((FAILED_CATEGORIES + 1)) + fi + echo "" +fi + +# Print summary +echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" +echo -e "${BLUE} Smoke Test Summary${NC}" +echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" +echo "" +echo " Total Categories: ${TOTAL_CATEGORIES}" +echo -e " ${GREEN}Passed: ${PASSED_CATEGORIES}${NC}" +if [ $FAILED_CATEGORIES -gt 0 ]; then + echo -e " ${RED}Failed: ${FAILED_CATEGORIES}${NC}" +else + echo " Failed: ${FAILED_CATEGORIES}" +fi +echo "" + +# Calculate percentage +if [ $TOTAL_CATEGORIES -gt 0 ]; then + PASS_PERCENTAGE=$((PASSED_CATEGORIES * 100 / TOTAL_CATEGORIES)) + echo " Pass Rate: ${PASS_PERCENTAGE}%" + echo "" +fi + +# Exit with appropriate code +if [ $FAILED_CATEGORIES -eq 0 ]; then + echo -e "${GREEN}βœ… All smoke tests passed!${NC}" + echo -e "${GREEN} System is ready for deployment.${NC}" + exit 0 +else + echo -e "${YELLOW}⚠️ Some smoke tests failed${NC}" + echo -e "${YELLOW} Review the failures above before deploying.${NC}" + exit 1 +fi diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 01dc92f51..5c51edaff 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -40,6 +40,13 @@ futures.workspace = true # gRPC and networking tonic.workspace = true +tonic-health = { version = "0.14", optional = true } + +# HTTP client for smoke tests +reqwest = { version = "0.12", features = ["json"], optional = true } + +# JWT for authentication tests +jsonwebtoken = { version = "9", optional = true } # Database and UUID sqlx.workspace = true @@ -101,6 +108,7 @@ memory-profiling = ["dhat", "jemalloc_pprof"] coverage-analysis = [] gpu-tests = [] integration-tests = ["redis", "influxdb2"] # Removed testcontainers +smoke-tests = ["reqwest", "tonic-health", "jsonwebtoken", "redis", "influxdb2"] # Performance optimization features simd = [] diff --git a/tests/smoke_tests.rs b/tests/smoke_tests.rs new file mode 100644 index 000000000..444287138 --- /dev/null +++ b/tests/smoke_tests.rs @@ -0,0 +1,18 @@ +//! Smoke Tests - Integration Test Entry Point +//! +//! This file serves as the integration test entry point for the smoke test suite. +//! Cargo will compile this as a separate test binary that can be run with: +//! +//! ```bash +//! cargo test --test smoke_tests +//! cargo test --test smoke_tests --features smoke-tests +//! ``` + +#[cfg(feature = "smoke-tests")] +mod smoke_tests; + +#[cfg(not(feature = "smoke-tests"))] +#[test] +fn smoke_tests_disabled() { + println!("⚠️ Smoke tests are disabled. Enable with: cargo test --features smoke-tests"); +} diff --git a/tests/smoke_tests/README.md b/tests/smoke_tests/README.md new file mode 100644 index 000000000..520915d11 --- /dev/null +++ b/tests/smoke_tests/README.md @@ -0,0 +1,359 @@ +# Smoke Tests - End-to-End System Validation + +## Overview + +Automated smoke test suite for validating complete Foxhunt HFT system functionality after deployment. These tests provide quick validation that all critical components are operational. + +## Test Categories + +### 1. Infrastructure Health (`infrastructure_health.rs`) +Tests core infrastructure services: +- **PostgreSQL**: Connection, schema validation, TimescaleDB extension +- **Redis**: Connection, SET/GET operations, key expiration +- **Vault**: Connectivity and health status +- **InfluxDB**: Ping endpoint and availability +- **Prometheus**: Health endpoint and metrics availability +- **Grafana**: API health check + +### 2. Service Health (`service_health.rs`) +Tests gRPC microservices: +- **Trading Service**: HTTP health check, gRPC connection (port 50052) +- **API Gateway**: gRPC connection (port 50051) +- **Backtesting Service**: gRPC connection (port 50053) - BLOCKED +- **ML Training Service**: gRPC connection (port 50054) - BLOCKED +- **Service Ports**: Validates all services are listening +- **Response Times**: Measures connection latency +- **Metrics Endpoints**: Validates Prometheus exporters (ports 9091-9094) + +### 3. Authentication Flow (`authentication_flow.rs`) +Tests authentication and security: +- **JWT Generation**: Token creation with claims +- **JWT Validation**: Signature verification and claim extraction +- **JWT Expiration**: Expired token rejection +- **JWT Signature**: Invalid signature detection +- **Session Storage**: Redis-based session management +- **Token Revocation**: Revocation list checking +- **Rate Limiting**: Request throttling with Redis + +### 4. Basic Order Flow (`basic_order_flow.rs`) +Tests core trading operations: +- **Order Submission**: Insert orders into PostgreSQL +- **Order Query**: Retrieve order by ID +- **Order Cancellation**: Update order status to cancelled +- **Position Management**: Create and query positions +- **Order History**: Query user's order history +- **Complete Lifecycle**: Full order flow from submission to cleanup + +## Usage + +### Run All Smoke Tests + +```bash +# Using the automated script (recommended) +./run_smoke_tests.sh + +# Using Cargo directly +cargo test --test smoke_tests --features smoke-tests +``` + +### Run Specific Categories + +```bash +# Infrastructure only +./run_smoke_tests.sh --category infrastructure + +# Service health only +./run_smoke_tests.sh --category service + +# Authentication only +./run_smoke_tests.sh --category authentication + +# Order flow only +./run_smoke_tests.sh --category order_flow +``` + +### Fast Mode (Critical Tests Only) + +```bash +# Run only infrastructure and service health checks +./run_smoke_tests.sh --fast +``` + +### Verbose Mode + +```bash +# Run with debug logging +./run_smoke_tests.sh --verbose +``` + +## Environment Variables + +All tests use environment variables with sensible defaults: + +```bash +# Infrastructure +DATABASE_URL=postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt +REDIS_URL=redis://localhost:6379 +VAULT_ADDR=http://localhost:8200 +INFLUXDB_URL=http://localhost:8086 + +# Services +API_GATEWAY_URL=http://localhost:50051 +TRADING_SERVICE_URL=http://localhost:50052 +BACKTESTING_SERVICE_URL=http://localhost:50053 +ML_TRAINING_SERVICE_URL=http://localhost:50054 + +# Monitoring +PROMETHEUS_URL=http://localhost:9090 +GRAFANA_URL=http://localhost:3000 + +# Authentication +JWT_SECRET=dev_secret_key_change_in_production + +# Logging +RUST_LOG=info # Set to 'debug' for verbose output +``` + +## Known Issues and Blockers + +### Blocked Tests (Agent 96 Findings) + +1. **Backtesting Service** - Configuration issues prevent deployment + - Test marked with `#[ignore]` attribute + - Skips gracefully when service unavailable + +2. **ML Training Service** - Configuration issues prevent deployment + - Test marked with `#[ignore]` attribute + - Skips gracefully when service unavailable + +### Working Tests + +- βœ… Infrastructure Health (PostgreSQL, Redis, Vault, InfluxDB, Prometheus, Grafana) +- βœ… Trading Service Health (confirmed working by Agent 96) +- βœ… API Gateway Health +- βœ… Authentication Flow (JWT, Sessions, Rate Limiting) +- βœ… Basic Order Flow (Database operations) + +## Test Architecture + +### Graceful Failure Handling + +Tests use the `skip_if_unavailable!` macro to handle service unavailability: + +```rust +skip_if_unavailable!("Service Name", { + // Test code here + result +}); +``` + +This allows tests to: +- Pass when services are available +- Skip gracefully when services are down (connection refused, timeout) +- Fail hard for actual test failures + +### Timeout Management + +All tests have configurable timeouts: +- **Standard smoke tests**: 10 seconds +- **Infrastructure tests**: 5 seconds + +Timeouts prevent hanging tests and provide quick feedback. + +### Parallel Execution + +Tests can run in parallel by default, but use `--test-threads=1` for sequential execution when debugging: + +```bash +cargo test --test smoke_tests -- --test-threads=1 --nocapture +``` + +## Integration with CI/CD + +### Docker Deployment Validation + +After deploying with Docker Compose: + +```bash +# 1. Start all services +docker-compose up -d + +# 2. Wait for health checks +docker-compose ps + +# 3. Run smoke tests +./run_smoke_tests.sh + +# 4. Check exit code +if [ $? -eq 0 ]; then + echo "Deployment validated - ready for production" +else + echo "Deployment issues detected - review logs" +fi +``` + +### Kubernetes Deployment Validation + +```bash +# 1. Deploy to cluster +kubectl apply -f k8s/ + +# 2. Wait for pods +kubectl wait --for=condition=ready pod -l app=foxhunt --timeout=300s + +# 3. Port forward services +kubectl port-forward svc/api-gateway 50051:50051 & +kubectl port-forward svc/trading-service 50052:50051 & + +# 4. Run smoke tests +./run_smoke_tests.sh + +# 5. Cleanup port forwards +pkill -f "kubectl port-forward" +``` + +## Metrics and Reporting + +### Test Pass Rate + +The smoke test runner reports: +- Total categories tested +- Passed/failed counts +- Pass percentage +- Exit code (0 = success, 1 = failure) + +### Sample Output + +``` +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Foxhunt HFT System - Smoke Test Suite +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Environment Configuration: + Database: postgresql://foxhunt:***@localhost:5432/foxhunt + Redis: redis://localhost:6379 + Vault: http://localhost:8200 + API Gateway: http://localhost:50051 + Trading Service: http://localhost:50052 + Log Level: info + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Testing: Infrastructure Health +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +βœ… PostgreSQL connection successful (TimescaleDB: true) +βœ… Redis connection and operations successful +βœ… Vault connectivity successful (status: 200) + +βœ… Infrastructure Health - PASSED + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Smoke Test Summary +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + + Total Categories: 4 + Passed: 4 + Failed: 0 + + Pass Rate: 100% + +βœ… All smoke tests passed! + System is ready for deployment. +``` + +## Future Enhancements + +### Planned Test Categories (Not Implemented) + +5. **Market Data Tests** (blocked by service availability) + - Subscribe to market data stream + - Receive quote updates + - Historical data queries + +6. **ML Inference Tests** (blocked by service availability) + - Model prediction requests + - Feature engineering pipeline + - Inference latency validation (<100ms) + +7. **Compliance Tests** (requires ML service) + - Audit log creation + - Best execution analysis + - Risk check validation + +8. **Advanced Monitoring** (requires full deployment) + - Alert manager connectivity + - Custom dashboard validation + - Log aggregation checks + +## Troubleshooting + +### Common Issues + +1. **Connection Refused** + ```bash + # Check if services are running + docker-compose ps + + # Restart failed services + docker-compose restart trading_service + ``` + +2. **Database Schema Missing** + ```bash + # Run migrations + cargo sqlx migrate run + ``` + +3. **Redis Connection Failed** + ```bash + # Check Redis is running + redis-cli ping + + # Restart Redis + docker-compose restart redis + ``` + +4. **JWT Secret Not Set** + ```bash + # Set JWT secret + export JWT_SECRET="your-secret-key-here" + ``` + +### Debug Mode + +Run tests with full debug output: + +```bash +RUST_LOG=debug ./run_smoke_tests.sh --verbose +``` + +This shows: +- Full test execution flow +- Connection attempts +- Query execution +- Error details + +## Contributing + +When adding new smoke tests: + +1. Create test file in `tests/smoke_tests/` +2. Add module to `tests/smoke_tests/mod.rs` +3. Use `skip_if_unavailable!` macro for graceful failures +4. Add timeout with `with_timeout()` wrapper +5. Update this README with test documentation +6. Update `run_smoke_tests.sh` with new category + +### Test Template + +```rust +#[tokio::test] +async fn test_new_feature() { + let result = with_timeout(async { + // Your test code here + Ok(()) + }).await; + + skip_if_unavailable!("Service Name", { result }); +} +``` diff --git a/tests/smoke_tests/authentication_flow.rs b/tests/smoke_tests/authentication_flow.rs new file mode 100644 index 000000000..f38b0ebd9 --- /dev/null +++ b/tests/smoke_tests/authentication_flow.rs @@ -0,0 +1,364 @@ +//! Authentication Flow Smoke Tests +//! +//! Tests to validate JWT authentication, session management, and authorization. + +use super::common::*; +use jsonwebtoken::{encode, decode, Header, Validation, EncodingKey, DecodingKey, Algorithm}; +use serde::{Deserialize, Serialize}; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[derive(Debug, Serialize, Deserialize)] +struct Claims { + sub: String, + iss: String, + aud: String, + exp: u64, + iat: u64, + roles: Vec, +} + +fn get_jwt_secret() -> String { + std::env::var("JWT_SECRET") + .unwrap_or_else(|_| "dev_secret_key_change_in_production".to_string()) +} + +fn create_test_token() -> Result> { + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + + let claims = Claims { + sub: "test_user".to_string(), + iss: "foxhunt-api-gateway".to_string(), + aud: "foxhunt-services".to_string(), + iat: now, + exp: now + 3600, // Valid for 1 hour + roles: vec!["trader".to_string()], + }; + + let token = encode( + &Header::new(Algorithm::HS256), + &claims, + &EncodingKey::from_secret(get_jwt_secret().as_bytes()), + )?; + + Ok(token) +} + +#[tokio::test] +async fn test_jwt_token_generation() { + let result = with_timeout(async { + let token = create_test_token() + .map_err(|e| Box::new(e) as Box)?; + + assert!(!token.is_empty(), "JWT token should not be empty"); + assert!(token.split('.').count() == 3, "JWT should have 3 parts"); + + println!("βœ… JWT token generation successful"); + println!(" Token length: {} characters", token.len()); + Ok(()) + }).await; + + assert!(result.is_ok(), "JWT token generation failed: {:?}", result.err()); +} + +#[tokio::test] +async fn test_jwt_token_validation() { + let result = with_timeout(async { + // Generate token + let token = create_test_token() + .map_err(|e| Box::new(e) as Box)?; + + // Validate token + let validation = Validation::new(Algorithm::HS256); + let token_data = decode::( + &token, + &DecodingKey::from_secret(get_jwt_secret().as_bytes()), + &validation, + ) + .map_err(|e| Box::new(e) as Box)?; + + assert_eq!(token_data.claims.sub, "test_user"); + assert_eq!(token_data.claims.iss, "foxhunt-api-gateway"); + assert_eq!(token_data.claims.aud, "foxhunt-services"); + assert!(token_data.claims.roles.contains(&"trader".to_string())); + + println!("βœ… JWT token validation successful"); + println!(" User: {}", token_data.claims.sub); + println!(" Roles: {:?}", token_data.claims.roles); + Ok(()) + }).await; + + assert!(result.is_ok(), "JWT token validation failed: {:?}", result.err()); +} + +#[tokio::test] +async fn test_jwt_token_expiration() { + let result = with_timeout(async { + // Create expired token + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + + let expired_claims = Claims { + sub: "test_user".to_string(), + iss: "foxhunt-api-gateway".to_string(), + aud: "foxhunt-services".to_string(), + iat: now - 7200, // Issued 2 hours ago + exp: now - 3600, // Expired 1 hour ago + roles: vec!["trader".to_string()], + }; + + let expired_token = encode( + &Header::new(Algorithm::HS256), + &expired_claims, + &EncodingKey::from_secret(get_jwt_secret().as_bytes()), + )?; + + // Try to validate expired token + let validation = Validation::new(Algorithm::HS256); + let result = decode::( + &expired_token, + &DecodingKey::from_secret(get_jwt_secret().as_bytes()), + &validation, + ); + + assert!(result.is_err(), "Expired token should not validate"); + + println!("βœ… JWT expiration validation successful"); + Ok(()) + }).await; + + assert!(result.is_ok(), "JWT expiration test failed: {:?}", result.err()); +} + +#[tokio::test] +async fn test_jwt_invalid_signature() { + let result = with_timeout(async { + // Create token with one secret + let token = create_test_token() + .map_err(|e| Box::new(e) as Box)?; + + // Try to validate with different secret + let validation = Validation::new(Algorithm::HS256); + let wrong_secret = "wrong_secret_key"; + let result = decode::( + &token, + &DecodingKey::from_secret(wrong_secret.as_bytes()), + &validation, + ); + + assert!(result.is_err(), "Token with invalid signature should not validate"); + + println!("βœ… JWT signature validation successful"); + Ok(()) + }).await; + + assert!(result.is_ok(), "JWT signature validation test failed: {:?}", result.err()); +} + +#[tokio::test] +async fn test_redis_session_storage() { + let result = with_timeout(async { + use redis::AsyncCommands; + + let client = redis::Client::open(redis_url().as_str()) + .map_err(|e| Box::new(e) as Box)?; + + let mut con = client.get_multiplexed_async_connection() + .await + .map_err(|e| Box::new(e) as Box)?; + + // Simulate session storage + let session_key = "smoke_test:session:test_user"; + let session_data = serde_json::json!({ + "user_id": "test_user", + "roles": ["trader"], + "created_at": chrono::Utc::now().to_rfc3339(), + }); + + // Store session with TTL + con.set_ex::<_, _, ()>(session_key, session_data.to_string(), 3600) + .await + .map_err(|e| Box::new(e) as Box)?; + + // Retrieve session + let retrieved: String = con.get(session_key) + .await + .map_err(|e| Box::new(e) as Box)?; + + let retrieved_json: serde_json::Value = serde_json::from_str(&retrieved)?; + assert_eq!(retrieved_json["user_id"], "test_user"); + + // Cleanup + con.del::<_, ()>(session_key) + .await + .map_err(|e| Box::new(e) as Box)?; + + println!("βœ… Redis session storage successful"); + Ok(()) + }).await; + + skip_if_unavailable!("Redis", { result }); +} + +#[tokio::test] +async fn test_jwt_revocation_check() { + let result = with_timeout(async { + use redis::AsyncCommands; + + let client = redis::Client::open(redis_url().as_str()) + .map_err(|e| Box::new(e) as Box)?; + + let mut con = client.get_multiplexed_async_connection() + .await + .map_err(|e| Box::new(e) as Box)?; + + // Create token + let token = create_test_token() + .map_err(|e| Box::new(e) as Box)?; + + // Simulate token revocation + let jti = "smoke_test_token_id"; + let revocation_key = format!("revoked_tokens:{}", jti); + + // Add to revocation list + con.set_ex::<_, _, ()>(&revocation_key, "revoked", 3600) + .await + .map_err(|e| Box::new(e) as Box)?; + + // Check if token is revoked + let is_revoked: bool = con.exists(&revocation_key) + .await + .map_err(|e| Box::new(e) as Box)?; + + assert!(is_revoked, "Token should be marked as revoked"); + + // Cleanup + con.del::<_, ()>(&revocation_key) + .await + .map_err(|e| Box::new(e) as Box)?; + + println!("βœ… JWT revocation check successful"); + Ok(()) + }).await; + + skip_if_unavailable!("Redis", { result }); +} + +#[tokio::test] +async fn test_rate_limiting() { + let result = with_timeout(async { + use redis::AsyncCommands; + + let client = redis::Client::open(redis_url().as_str()) + .map_err(|e| Box::new(e) as Box)?; + + let mut con = client.get_multiplexed_async_connection() + .await + .map_err(|e| Box::new(e) as Box)?; + + // Simulate rate limiting + let rate_limit_key = "smoke_test:rate_limit:test_user"; + let window_seconds = 60; + let max_requests = 100; + + // Increment counter + let count: i32 = con.incr(&rate_limit_key, 1) + .await + .map_err(|e| Box::new(e) as Box)?; + + if count == 1 { + // Set expiration on first request + con.expire::<_, ()>(&rate_limit_key, window_seconds) + .await + .map_err(|e| Box::new(e) as Box)?; + } + + // Check if rate limit exceeded + let is_limited = count > max_requests; + assert!(!is_limited, "Rate limit should not be exceeded on first request"); + + // Get TTL + let ttl: i32 = con.ttl(&rate_limit_key) + .await + .map_err(|e| Box::new(e) as Box)?; + + assert!(ttl > 0 && ttl <= window_seconds, "TTL should be set correctly"); + + // Cleanup + con.del::<_, ()>(&rate_limit_key) + .await + .map_err(|e| Box::new(e) as Box)?; + + println!("βœ… Rate limiting check successful"); + println!(" Request count: {}/{}", count, max_requests); + println!(" TTL: {} seconds", ttl); + Ok(()) + }).await; + + skip_if_unavailable!("Redis", { result }); +} + +#[tokio::test] +async fn test_authentication_flow_complete() { + println!("πŸ” Running complete authentication flow..."); + + let mut steps_passed = 0; + let total_steps = 4; + + // Step 1: Generate token + if create_test_token().is_ok() { + println!(" βœ“ JWT token generation"); + steps_passed += 1; + } else { + println!(" βœ— JWT token generation failed"); + } + + // Step 2: Validate token + if let Ok(token) = create_test_token() { + let validation = Validation::new(Algorithm::HS256); + if decode::( + &token, + &DecodingKey::from_secret(get_jwt_secret().as_bytes()), + &validation, + ).is_ok() { + println!(" βœ“ JWT token validation"); + steps_passed += 1; + } else { + println!(" βœ— JWT token validation failed"); + } + } + + // Step 3: Session storage + if let Ok(client) = redis::Client::open(redis_url().as_str()) { + if let Ok(mut con) = client.get_multiplexed_async_connection().await { + use redis::AsyncCommands; + let test_key = "smoke_test:auth_flow:session"; + if con.set_ex::<_, _, ()>(test_key, "test", 60).await.is_ok() { + println!(" βœ“ Session storage"); + steps_passed += 1; + let _ = con.del::<_, ()>(test_key).await; + } + } + } else { + println!(" ⚠ Session storage (Redis unavailable)"); + } + + // Step 4: Rate limiting + if let Ok(client) = redis::Client::open(redis_url().as_str()) { + if let Ok(mut con) = client.get_multiplexed_async_connection().await { + use redis::AsyncCommands; + let test_key = "smoke_test:auth_flow:rate_limit"; + if con.incr::<_, _, i32>(test_key, 1).await.is_ok() { + println!(" βœ“ Rate limiting"); + steps_passed += 1; + let _ = con.del::<_, ()>(test_key).await; + } + } + } else { + println!(" ⚠ Rate limiting (Redis unavailable)"); + } + + println!("\nπŸ“Š Authentication Flow Summary:"); + println!(" Steps passed: {}/{}", steps_passed, total_steps); + + assert!(steps_passed >= 2, "Authentication flow requires at least JWT functionality"); + println!("βœ… Authentication flow check complete"); +} diff --git a/tests/smoke_tests/basic_order_flow.rs b/tests/smoke_tests/basic_order_flow.rs new file mode 100644 index 000000000..545e411b7 --- /dev/null +++ b/tests/smoke_tests/basic_order_flow.rs @@ -0,0 +1,413 @@ +//! Basic Order Flow Smoke Tests +//! +//! Tests to validate core trading functionality including order submission, +//! querying, cancellation, and position management. + +use super::common::*; + +#[tokio::test] +async fn test_database_order_submission() { + let result = with_timeout(async { + let pool = sqlx::PgPool::connect(&database_url()) + .await + .map_err(|e| Box::new(e) as Box)?; + + // Create a test order + let order_id = uuid::Uuid::new_v4(); + let user_id = "smoke_test_user"; + let symbol = "BTC-USD"; + let order_type = "market"; + let side = "buy"; + let quantity = 1.0; + let price = None::; + + // Insert order + sqlx::query( + r#" + INSERT INTO orders ( + id, user_id, symbol, order_type, side, quantity, price, + status, created_at, updated_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW(), NOW()) + "# + ) + .bind(&order_id) + .bind(user_id) + .bind(symbol) + .bind(order_type) + .bind(side) + .bind(quantity) + .bind(price) + .bind("pending") + .execute(&pool) + .await + .map_err(|e| Box::new(e) as Box)?; + + println!("βœ… Order submitted to database"); + println!(" Order ID: {}", order_id); + println!(" Symbol: {}", symbol); + println!(" Side: {}", side); + println!(" Quantity: {}", quantity); + + // Cleanup + sqlx::query("DELETE FROM orders WHERE id = $1") + .bind(&order_id) + .execute(&pool) + .await + .map_err(|e| Box::new(e) as Box)?; + + pool.close().await; + Ok(()) + }).await; + + skip_if_unavailable!("PostgreSQL", { result }); +} + +#[tokio::test] +async fn test_database_order_query() { + let result = with_timeout(async { + let pool = sqlx::PgPool::connect(&database_url()) + .await + .map_err(|e| Box::new(e) as Box)?; + + // Create a test order + let order_id = uuid::Uuid::new_v4(); + let user_id = "smoke_test_user"; + let symbol = "ETH-USD"; + + sqlx::query( + r#" + INSERT INTO orders ( + id, user_id, symbol, order_type, side, quantity, + status, created_at, updated_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, NOW(), NOW()) + "# + ) + .bind(&order_id) + .bind(user_id) + .bind(symbol) + .bind("limit") + .bind("sell") + .bind(2.0) + .bind("pending") + .execute(&pool) + .await + .map_err(|e| Box::new(e) as Box)?; + + // Query order + let order: (uuid::Uuid, String, String) = sqlx::query_as( + "SELECT id, user_id, symbol FROM orders WHERE id = $1" + ) + .bind(&order_id) + .fetch_one(&pool) + .await + .map_err(|e| Box::new(e) as Box)?; + + assert_eq!(order.0, order_id); + assert_eq!(order.1, user_id); + assert_eq!(order.2, symbol); + + println!("βœ… Order query successful"); + println!(" Order ID: {}", order.0); + println!(" User ID: {}", order.1); + println!(" Symbol: {}", order.2); + + // Cleanup + sqlx::query("DELETE FROM orders WHERE id = $1") + .bind(&order_id) + .execute(&pool) + .await + .map_err(|e| Box::new(e) as Box)?; + + pool.close().await; + Ok(()) + }).await; + + skip_if_unavailable!("PostgreSQL", { result }); +} + +#[tokio::test] +async fn test_database_order_cancellation() { + let result = with_timeout(async { + let pool = sqlx::PgPool::connect(&database_url()) + .await + .map_err(|e| Box::new(e) as Box)?; + + // Create a test order + let order_id = uuid::Uuid::new_v4(); + + sqlx::query( + r#" + INSERT INTO orders ( + id, user_id, symbol, order_type, side, quantity, + status, created_at, updated_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, NOW(), NOW()) + "# + ) + .bind(&order_id) + .bind("smoke_test_user") + .bind("BTC-USD") + .bind("limit") + .bind("buy") + .bind(0.5) + .bind("pending") + .execute(&pool) + .await + .map_err(|e| Box::new(e) as Box)?; + + // Cancel order + sqlx::query( + "UPDATE orders SET status = $1, updated_at = NOW() WHERE id = $2" + ) + .bind("cancelled") + .bind(&order_id) + .execute(&pool) + .await + .map_err(|e| Box::new(e) as Box)?; + + // Verify cancellation + let status: (String,) = sqlx::query_as( + "SELECT status FROM orders WHERE id = $1" + ) + .bind(&order_id) + .fetch_one(&pool) + .await + .map_err(|e| Box::new(e) as Box)?; + + assert_eq!(status.0, "cancelled"); + + println!("βœ… Order cancellation successful"); + println!(" Order ID: {}", order_id); + println!(" Status: {}", status.0); + + // Cleanup + sqlx::query("DELETE FROM orders WHERE id = $1") + .bind(&order_id) + .execute(&pool) + .await + .map_err(|e| Box::new(e) as Box)?; + + pool.close().await; + Ok(()) + }).await; + + skip_if_unavailable!("PostgreSQL", { result }); +} + +#[tokio::test] +async fn test_database_position_management() { + let result = with_timeout(async { + let pool = sqlx::PgPool::connect(&database_url()) + .await + .map_err(|e| Box::new(e) as Box)?; + + let position_id = uuid::Uuid::new_v4(); + let user_id = "smoke_test_user"; + let symbol = "BTC-USD"; + let quantity = 1.5; + let entry_price = 50000.0; + + // Create position + sqlx::query( + r#" + INSERT INTO positions ( + id, user_id, symbol, quantity, entry_price, current_price, + unrealized_pnl, created_at, updated_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, NOW(), NOW()) + "# + ) + .bind(&position_id) + .bind(user_id) + .bind(symbol) + .bind(quantity) + .bind(entry_price) + .bind(entry_price) // current_price = entry_price initially + .bind(0.0) // unrealized_pnl = 0 initially + .execute(&pool) + .await + .map_err(|e| Box::new(e) as Box)?; + + // Query position + let position: (uuid::Uuid, String, f64, f64) = sqlx::query_as( + "SELECT id, symbol, quantity, entry_price FROM positions WHERE id = $1" + ) + .bind(&position_id) + .fetch_one(&pool) + .await + .map_err(|e| Box::new(e) as Box)?; + + assert_eq!(position.0, position_id); + assert_eq!(position.1, symbol); + assert_eq!(position.2, quantity); + assert_eq!(position.3, entry_price); + + println!("βœ… Position management successful"); + println!(" Position ID: {}", position.0); + println!(" Symbol: {}", position.1); + println!(" Quantity: {}", position.2); + println!(" Entry Price: ${}", position.3); + + // Cleanup + sqlx::query("DELETE FROM positions WHERE id = $1") + .bind(&position_id) + .execute(&pool) + .await + .map_err(|e| Box::new(e) as Box)?; + + pool.close().await; + Ok(()) + }).await; + + skip_if_unavailable!("PostgreSQL", { result }); +} + +#[tokio::test] +async fn test_database_order_history() { + let result = with_timeout(async { + let pool = sqlx::PgPool::connect(&database_url()) + .await + .map_err(|e| Box::new(e) as Box)?; + + let user_id = "smoke_test_user"; + + // Create multiple test orders + let order_ids: Vec = (0..3) + .map(|_| uuid::Uuid::new_v4()) + .collect(); + + for order_id in &order_ids { + sqlx::query( + r#" + INSERT INTO orders ( + id, user_id, symbol, order_type, side, quantity, + status, created_at, updated_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, NOW(), NOW()) + "# + ) + .bind(order_id) + .bind(user_id) + .bind("BTC-USD") + .bind("market") + .bind("buy") + .bind(0.1) + .bind("filled") + .execute(&pool) + .await + .map_err(|e| Box::new(e) as Box)?; + } + + // Query order history + let orders: Vec<(uuid::Uuid, String)> = sqlx::query_as( + "SELECT id, status FROM orders WHERE user_id = $1 ORDER BY created_at DESC LIMIT 10" + ) + .bind(user_id) + .fetch_all(&pool) + .await + .map_err(|e| Box::new(e) as Box)?; + + assert!(orders.len() >= 3, "Should find at least 3 orders"); + + println!("βœ… Order history query successful"); + println!(" Found {} orders for user {}", orders.len(), user_id); + + // Cleanup + for order_id in &order_ids { + sqlx::query("DELETE FROM orders WHERE id = $1") + .bind(order_id) + .execute(&pool) + .await + .map_err(|e| Box::new(e) as Box)?; + } + + pool.close().await; + Ok(()) + }).await; + + skip_if_unavailable!("PostgreSQL", { result }); +} + +#[tokio::test] +async fn test_complete_order_lifecycle() { + println!("πŸ”„ Running complete order lifecycle..."); + + let mut steps_passed = 0; + let total_steps = 4; + + if let Ok(pool) = sqlx::PgPool::connect(&database_url()).await { + let order_id = uuid::Uuid::new_v4(); + + // Step 1: Submit order + if sqlx::query( + r#" + INSERT INTO orders ( + id, user_id, symbol, order_type, side, quantity, + status, created_at, updated_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, NOW(), NOW()) + "# + ) + .bind(&order_id) + .bind("smoke_test_user") + .bind("BTC-USD") + .bind("market") + .bind("buy") + .bind(1.0) + .bind("pending") + .execute(&pool) + .await + .is_ok() { + println!(" βœ“ Order submission"); + steps_passed += 1; + } + + // Step 2: Query order + if sqlx::query_as::<_, (String,)>( + "SELECT status FROM orders WHERE id = $1" + ) + .bind(&order_id) + .fetch_one(&pool) + .await + .is_ok() { + println!(" βœ“ Order query"); + steps_passed += 1; + } + + // Step 3: Update order + if sqlx::query( + "UPDATE orders SET status = $1, updated_at = NOW() WHERE id = $2" + ) + .bind("filled") + .bind(&order_id) + .execute(&pool) + .await + .is_ok() { + println!(" βœ“ Order update"); + steps_passed += 1; + } + + // Step 4: Cancel/cleanup + if sqlx::query("DELETE FROM orders WHERE id = $1") + .bind(&order_id) + .execute(&pool) + .await + .is_ok() { + println!(" βœ“ Order cleanup"); + steps_passed += 1; + } + + pool.close().await; + } else { + println!(" ⚠ Database unavailable - skipping order lifecycle test"); + return; + } + + println!("\nπŸ“Š Order Lifecycle Summary:"); + println!(" Steps passed: {}/{}", steps_passed, total_steps); + + assert_eq!(steps_passed, total_steps, "Complete order lifecycle should pass all steps"); + println!("βœ… Order lifecycle check complete"); +} diff --git a/tests/smoke_tests/infrastructure_health.rs b/tests/smoke_tests/infrastructure_health.rs new file mode 100644 index 000000000..cc8f08c48 --- /dev/null +++ b/tests/smoke_tests/infrastructure_health.rs @@ -0,0 +1,293 @@ +//! Infrastructure Health Smoke Tests +//! +//! Tests to validate that core infrastructure services are available and functional. +//! These tests run first to ensure the foundation is solid before testing application services. + +use super::common::*; + +#[tokio::test] +async fn test_postgres_connection() { + let result = with_timeout(async { + let pool = sqlx::PgPool::connect(&database_url()) + .await + .map_err(|e| Box::new(e) as Box)?; + + // Verify database is responsive + let row: (i32,) = sqlx::query_as("SELECT 1") + .fetch_one(&pool) + .await + .map_err(|e| Box::new(e) as Box)?; + + assert_eq!(row.0, 1, "Database query returned unexpected value"); + + // Check TimescaleDB extension + let has_timescale: (bool,) = sqlx::query_as( + "SELECT COUNT(*) > 0 FROM pg_extension WHERE extname = 'timescaledb'" + ) + .fetch_one(&pool) + .await + .map_err(|e| Box::new(e) as Box)?; + + pool.close().await; + + println!("βœ… PostgreSQL connection successful (TimescaleDB: {})", has_timescale.0); + Ok(()) + }).await; + + skip_if_unavailable!("PostgreSQL", { result }); +} + +#[tokio::test] +async fn test_postgres_schema_exists() { + let result = with_timeout(async { + let pool = sqlx::PgPool::connect(&database_url()) + .await + .map_err(|e| Box::new(e) as Box)?; + + // Check for key tables + let tables = vec!["orders", "positions", "market_data", "audit_log"]; + for table in tables { + let exists: (bool,) = sqlx::query_as( + "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = $1)" + ) + .bind(table) + .fetch_one(&pool) + .await + .map_err(|e| Box::new(e) as Box)?; + + assert!(exists.0, "Table '{}' does not exist", table); + println!(" βœ“ Table '{}' exists", table); + } + + pool.close().await; + println!("βœ… PostgreSQL schema validation successful"); + Ok(()) + }).await; + + skip_if_unavailable!("PostgreSQL", { result }); +} + +#[tokio::test] +async fn test_redis_connection() { + let result = with_timeout(async { + use redis::AsyncCommands; + + let client = redis::Client::open(redis_url().as_str()) + .map_err(|e| Box::new(e) as Box)?; + + let mut con = client.get_multiplexed_async_connection() + .await + .map_err(|e| Box::new(e) as Box)?; + + // Test PING + let pong: String = redis::cmd("PING") + .query_async(&mut con) + .await + .map_err(|e| Box::new(e) as Box)?; + + assert_eq!(pong, "PONG", "Redis PING returned unexpected value"); + + // Test SET/GET + let test_key = "smoke_test:redis_health"; + let test_value = "healthy"; + + con.set::<_, _, ()>(test_key, test_value) + .await + .map_err(|e| Box::new(e) as Box)?; + + let retrieved: String = con.get(test_key) + .await + .map_err(|e| Box::new(e) as Box)?; + + assert_eq!(retrieved, test_value, "Redis GET returned unexpected value"); + + // Cleanup + con.del::<_, ()>(test_key) + .await + .map_err(|e| Box::new(e) as Box)?; + + println!("βœ… Redis connection and operations successful"); + Ok(()) + }).await; + + skip_if_unavailable!("Redis", { result }); +} + +#[tokio::test] +async fn test_vault_connectivity() { + let result = with_timeout(async { + let client = reqwest::Client::new(); + let vault_url = vault_url(); + + // Test Vault health endpoint + let response = client + .get(format!("{}/v1/sys/health", vault_url)) + .send() + .await + .map_err(|e| Box::new(e) as Box)?; + + // Vault health returns 200 (initialized and unsealed), 429 (unsealed and standby), + // 472 (data recovery mode), 473 (performance standby), or 501 (not initialized) + let status = response.status(); + assert!( + status == 200 || status == 429 || status == 473, + "Vault health check returned unexpected status: {}", + status + ); + + println!("βœ… Vault connectivity successful (status: {})", status); + Ok(()) + }).await; + + skip_if_unavailable!("Vault", { result }); +} + +#[tokio::test] +async fn test_influxdb_connectivity() { + let result = with_timeout(async { + let client = reqwest::Client::new(); + let influxdb_url = influxdb_url(); + + // Test InfluxDB ping endpoint + let response = client + .get(format!("{}/ping", influxdb_url)) + .send() + .await + .map_err(|e| Box::new(e) as Box)?; + + assert!( + response.status().is_success(), + "InfluxDB ping failed with status: {}", + response.status() + ); + + println!("βœ… InfluxDB connectivity successful"); + Ok(()) + }).await; + + skip_if_unavailable!("InfluxDB", { result }); +} + +#[tokio::test] +async fn test_prometheus_connectivity() { + let result = with_timeout(async { + let client = reqwest::Client::new(); + let prometheus_url = prometheus_url(); + + // Test Prometheus health endpoint + let response = client + .get(format!("{}/-/healthy", prometheus_url)) + .send() + .await + .map_err(|e| Box::new(e) as Box)?; + + assert!( + response.status().is_success(), + "Prometheus health check failed with status: {}", + response.status() + ); + + println!("βœ… Prometheus connectivity successful"); + Ok(()) + }).await; + + skip_if_unavailable!("Prometheus", { result }); +} + +#[tokio::test] +async fn test_grafana_connectivity() { + let result = with_timeout(async { + let client = reqwest::Client::new(); + let grafana_url = grafana_url(); + + // Test Grafana API health endpoint + let response = client + .get(format!("{}/api/health", grafana_url)) + .send() + .await + .map_err(|e| Box::new(e) as Box)?; + + assert!( + response.status().is_success(), + "Grafana health check failed with status: {}", + response.status() + ); + + println!("βœ… Grafana connectivity successful"); + Ok(()) + }).await; + + skip_if_unavailable!("Grafana", { result }); +} + +#[tokio::test] +async fn test_infrastructure_all_healthy() { + println!("πŸ₯ Running comprehensive infrastructure health check..."); + + let mut checks_passed = 0; + let mut checks_failed = 0; + + // PostgreSQL + match sqlx::PgPool::connect(&database_url()).await { + Ok(pool) => { + pool.close().await; + println!(" βœ“ PostgreSQL"); + checks_passed += 1; + } + Err(e) => { + eprintln!(" βœ— PostgreSQL: {}", e); + checks_failed += 1; + } + } + + // Redis + match redis::Client::open(redis_url().as_str()) { + Ok(client) => { + match client.get_multiplexed_async_connection().await { + Ok(_) => { + println!(" βœ“ Redis"); + checks_passed += 1; + } + Err(e) => { + eprintln!(" βœ— Redis: {}", e); + checks_failed += 1; + } + } + } + Err(e) => { + eprintln!(" βœ— Redis: {}", e); + checks_failed += 1; + } + } + + // Vault (optional - don't fail if unavailable) + let client = reqwest::Client::new(); + match client.get(format!("{}/v1/sys/health", vault_url())).send().await { + Ok(response) if response.status() == 200 || response.status() == 429 => { + println!(" βœ“ Vault"); + checks_passed += 1; + } + _ => { + println!(" ⚠ Vault (not critical)"); + } + } + + // InfluxDB (optional - don't fail if unavailable) + match client.get(format!("{}/ping", influxdb_url())).send().await { + Ok(response) if response.status().is_success() => { + println!(" βœ“ InfluxDB"); + checks_passed += 1; + } + _ => { + println!(" ⚠ InfluxDB (not critical)"); + } + } + + println!("\nπŸ“Š Infrastructure Health Summary:"); + println!(" Passed: {}", checks_passed); + println!(" Failed: {}", checks_failed); + + // Require at least PostgreSQL and Redis + assert!(checks_passed >= 2, "Critical infrastructure services are down"); + println!("βœ… Infrastructure health check complete"); +} diff --git a/tests/smoke_tests/mod.rs b/tests/smoke_tests/mod.rs new file mode 100644 index 000000000..9903f4788 --- /dev/null +++ b/tests/smoke_tests/mod.rs @@ -0,0 +1,134 @@ +//! Smoke Tests - End-to-End System Validation +//! +//! Automated smoke tests to validate complete system functionality after deployment. +//! Tests are organized by category and can run selectively based on service availability. +//! +//! ## Test Categories +//! - Infrastructure Health (PostgreSQL, Redis, Vault, InfluxDB) +//! - Service Health (API Gateway, Trading, Backtesting, ML Training) +//! - Authentication Flow (JWT, MFA, Session Management) +//! - Basic Order Flow (Submit, Query, Cancel, Positions) +//! - Market Data (Streaming, Historical Queries) +//! - ML Inference (Predictions, Feature Engineering) +//! - Compliance (Audit Logs, Best Execution, Risk Checks) +//! - Metrics and Monitoring (Prometheus, Grafana) +//! +//! ## Usage +//! ```bash +//! # Run all smoke tests +//! cargo test --test smoke_tests +//! +//! # Run specific category +//! cargo test --test smoke_tests infrastructure_health +//! cargo test --test smoke_tests service_health +//! cargo test --test smoke_tests basic_order_flow +//! +//! # Run with logging +//! RUST_LOG=debug cargo test --test smoke_tests -- --nocapture +//! ``` + +pub mod infrastructure_health; +pub mod service_health; +pub mod authentication_flow; +pub mod basic_order_flow; + +/// Common test utilities and helpers +pub mod common { + use std::time::Duration; + use tokio::time::timeout; + + /// Standard timeout for smoke tests (10 seconds) + pub const SMOKE_TEST_TIMEOUT: Duration = Duration::from_secs(10); + + /// Standard timeout for infrastructure checks (5 seconds) + pub const INFRA_TEST_TIMEOUT: Duration = Duration::from_secs(5); + + /// Load database URL from environment + pub fn database_url() -> String { + std::env::var("DATABASE_URL") + .unwrap_or_else(|_| "postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt".to_string()) + } + + /// Load Redis URL from environment + pub fn redis_url() -> String { + std::env::var("REDIS_URL") + .unwrap_or_else(|_| "redis://localhost:6379".to_string()) + } + + /// Load Vault URL from environment + pub fn vault_url() -> String { + std::env::var("VAULT_ADDR") + .unwrap_or_else(|_| "http://localhost:8200".to_string()) + } + + /// Load InfluxDB URL from environment + pub fn influxdb_url() -> String { + std::env::var("INFLUXDB_URL") + .unwrap_or_else(|_| "http://localhost:8086".to_string()) + } + + /// Load API Gateway URL from environment + pub fn api_gateway_url() -> String { + std::env::var("API_GATEWAY_URL") + .unwrap_or_else(|_| "http://localhost:50051".to_string()) + } + + /// Load Trading Service URL from environment + pub fn trading_service_url() -> String { + std::env::var("TRADING_SERVICE_URL") + .unwrap_or_else(|_| "http://localhost:50052".to_string()) + } + + /// Load Backtesting Service URL from environment + pub fn backtesting_service_url() -> String { + std::env::var("BACKTESTING_SERVICE_URL") + .unwrap_or_else(|_| "http://localhost:50053".to_string()) + } + + /// Load ML Training Service URL from environment + pub fn ml_training_service_url() -> String { + std::env::var("ML_TRAINING_SERVICE_URL") + .unwrap_or_else(|_| "http://localhost:50054".to_string()) + } + + /// Load Prometheus URL from environment + pub fn prometheus_url() -> String { + std::env::var("PROMETHEUS_URL") + .unwrap_or_else(|_| "http://localhost:9090".to_string()) + } + + /// Load Grafana URL from environment + pub fn grafana_url() -> String { + std::env::var("GRAFANA_URL") + .unwrap_or_else(|_| "http://localhost:3000".to_string()) + } + + /// Run a test with timeout + pub async fn with_timeout(future: F) -> Result> + where + F: std::future::Future>>, + { + match timeout(SMOKE_TEST_TIMEOUT, future).await { + Ok(result) => result, + Err(_) => Err("Test timed out after 10 seconds".into()), + } + } + + /// Skip test if service is not available + /// This allows tests to be skipped gracefully when services are down + #[macro_export] + macro_rules! skip_if_unavailable { + ($service:expr, $test:block) => { + match $test { + Ok(result) => result, + Err(e) if e.to_string().contains("connection refused") || + e.to_string().contains("connection reset") || + e.to_string().contains("timed out") => { + eprintln!("⏭️ Skipping test - {} not available: {}", $service, e); + return; + } + Err(e) => panic!("Test failed: {}", e), + } + }; + } +} diff --git a/tests/smoke_tests/service_health.rs b/tests/smoke_tests/service_health.rs new file mode 100644 index 000000000..2aff44493 --- /dev/null +++ b/tests/smoke_tests/service_health.rs @@ -0,0 +1,265 @@ +//! Service Health Smoke Tests +//! +//! Tests to validate that gRPC services are running and responsive. +//! Uses gRPC health check protocol to verify service availability. + +use super::common::*; + +#[tokio::test] +async fn test_trading_service_health() { + let result = with_timeout(async { + let url = trading_service_url(); + println!("Testing Trading Service at {}", url); + + // Try HTTP health endpoint first (faster) + let client = reqwest::Client::new(); + let http_health_url = url.replace("50052", "8080").replace("http://", "http://"); + + match client.get(format!("{}/health", http_health_url)).send().await { + Ok(response) if response.status().is_success() => { + println!("βœ… Trading Service HTTP health check passed"); + return Ok(()); + } + _ => { + println!(" HTTP health check not available, trying gRPC..."); + } + } + + // Fallback to gRPC connection test + // Note: We can't use tonic_health without the proto definitions, + // so we'll just test if we can connect + match tonic::transport::Channel::from_shared(url.clone()) { + Ok(endpoint) => { + match endpoint.connect().await { + Ok(_channel) => { + println!("βœ… Trading Service gRPC connection successful"); + Ok(()) + } + Err(e) => Err(Box::new(e) as Box), + } + } + Err(e) => Err(Box::new(e) as Box), + } + }).await; + + skip_if_unavailable!("Trading Service", { result }); +} + +#[tokio::test] +async fn test_api_gateway_health() { + let result = with_timeout(async { + let url = api_gateway_url(); + println!("Testing API Gateway at {}", url); + + // Try gRPC connection + match tonic::transport::Channel::from_shared(url.clone()) { + Ok(endpoint) => { + match endpoint.connect().await { + Ok(_channel) => { + println!("βœ… API Gateway gRPC connection successful"); + Ok(()) + } + Err(e) => Err(Box::new(e) as Box), + } + } + Err(e) => Err(Box::new(e) as Box), + } + }).await; + + skip_if_unavailable!("API Gateway", { result }); +} + +#[tokio::test] +#[ignore = "Backtesting Service may not be available due to configuration issues"] +async fn test_backtesting_service_health() { + let result = with_timeout(async { + let url = backtesting_service_url(); + println!("Testing Backtesting Service at {}", url); + + match tonic::transport::Channel::from_shared(url.clone()) { + Ok(endpoint) => { + match endpoint.connect().await { + Ok(_channel) => { + println!("βœ… Backtesting Service gRPC connection successful"); + Ok(()) + } + Err(e) => Err(Box::new(e) as Box), + } + } + Err(e) => Err(Box::new(e) as Box), + } + }).await; + + skip_if_unavailable!("Backtesting Service", { result }); +} + +#[tokio::test] +#[ignore = "ML Training Service may not be available due to configuration issues"] +async fn test_ml_training_service_health() { + let result = with_timeout(async { + let url = ml_training_service_url(); + println!("Testing ML Training Service at {}", url); + + match tonic::transport::Channel::from_shared(url.clone()) { + Ok(endpoint) => { + match endpoint.connect().await { + Ok(_channel) => { + println!("βœ… ML Training Service gRPC connection successful"); + Ok(()) + } + Err(e) => Err(Box::new(e) as Box), + } + } + Err(e) => Err(Box::new(e) as Box), + } + }).await; + + skip_if_unavailable!("ML Training Service", { result }); +} + +#[tokio::test] +async fn test_service_ports_listening() { + println!("πŸ” Checking service ports..."); + + let ports_to_check = vec![ + (50051, "API Gateway"), + (50052, "Trading Service"), + (50053, "Backtesting Service"), + (50054, "ML Training Service"), + ]; + + let mut listening = 0; + let mut not_listening = 0; + + for (port, service) in ports_to_check { + match std::net::TcpStream::connect(format!("localhost:{}", port)) { + Ok(_) => { + println!(" βœ“ Port {} ({}) is listening", port, service); + listening += 1; + } + Err(_) => { + println!(" βœ— Port {} ({}) is not listening", port, service); + not_listening += 1; + } + } + } + + println!("\nπŸ“Š Service Port Summary:"); + println!(" Listening: {}", listening); + println!(" Not listening: {}", not_listening); + + // Require at least Trading Service to be up + assert!(listening >= 1, "No services are listening"); + println!("βœ… Service port check complete"); +} + +#[tokio::test] +async fn test_service_response_times() { + println!("⏱️ Measuring service response times..."); + + let services = vec![ + (trading_service_url(), "Trading Service"), + (api_gateway_url(), "API Gateway"), + ]; + + for (url, name) in services { + let start = std::time::Instant::now(); + + match tonic::transport::Channel::from_shared(url.clone()) { + Ok(endpoint) => { + match endpoint.connect().await { + Ok(_channel) => { + let elapsed = start.elapsed(); + println!(" βœ“ {} connected in {:?}", name, elapsed); + + // Warn if connection takes too long + if elapsed.as_millis() > 1000 { + println!(" ⚠ Connection took longer than expected"); + } + } + Err(_) => { + println!(" ⚠ {} not available", name); + } + } + } + Err(_) => { + println!(" ⚠ {} invalid URL", name); + } + } + } + + println!("βœ… Service response time check complete"); +} + +#[tokio::test] +async fn test_metrics_endpoints() { + println!("πŸ“Š Checking metrics endpoints..."); + + let client = reqwest::Client::new(); + let metrics_ports = vec![ + (9091, "API Gateway"), + (9092, "Trading Service"), + (9093, "Backtesting Service"), + (9094, "ML Training Service"), + ]; + + let mut available = 0; + let mut unavailable = 0; + + for (port, service) in metrics_ports { + let url = format!("http://localhost:{}/metrics", port); + + match client.get(&url).send().await { + Ok(response) if response.status().is_success() => { + println!(" βœ“ {} metrics available at port {}", service, port); + available += 1; + + // Verify Prometheus format + if let Ok(text) = response.text().await { + assert!( + text.contains("# HELP") || text.contains("# TYPE"), + "{} metrics not in Prometheus format", + service + ); + } + } + _ => { + println!(" ⚠ {} metrics not available at port {}", service, port); + unavailable += 1; + } + } + } + + println!("\nπŸ“Š Metrics Endpoint Summary:"); + println!(" Available: {}", available); + println!(" Unavailable: {}", unavailable); + + println!("βœ… Metrics endpoint check complete"); +} + +#[tokio::test] +async fn test_service_versions() { + println!("πŸ”– Checking service versions..."); + + // Try to get version info from Trading Service health endpoint + let client = reqwest::Client::new(); + let health_url = "http://localhost:8080/health"; + + match client.get(health_url).send().await { + Ok(response) if response.status().is_success() => { + if let Ok(json) = response.json::().await { + if let Some(version) = json.get("version") { + println!(" βœ“ Trading Service version: {}", version); + } + if let Some(service) = json.get("service") { + println!(" βœ“ Service name: {}", service); + } + } + } + _ => { + println!(" ⚠ Could not retrieve version information"); + } + } + + println!("βœ… Service version check complete"); +}