foxhunt/deployment/scripts/health_check.sh

#!/bin/bash
# Comprehensive health check for Foxhunt HFT Trading System
# Version: 3.0.0
# Wave 75 Agent 9

set -euo pipefail

echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "       FOXHUNT HFT PRODUCTION HEALTH CHECK"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "       $(date '+%Y-%m-%d %H:%M:%S')"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

CHECKS_PASSED=0
TOTAL_CHECKS=12

# Database connection (optional, from environment)
DATABASE_URL="${DATABASE_URL:-postgresql://localhost:5432/foxhunt_production}"

# Function to check service health
check_service() {
    local service_name=$1
    local port=$2
    local health_url=$3

    echo -n "Checking $service_name... "

    # Check if port is listening
    if nc -z localhost $port 2>/dev/null; then
        echo -n "port ✅ "
    else
        echo "❌ (port $port not listening)"
        return 1
    fi

    # Check health endpoint (if provided)
    if [ -n "$health_url" ]; then
        if curl -s -f "$health_url" > /dev/null 2>&1; then
            echo "health ✅"
        else
            echo "⚠️ (health endpoint failed)"
            return 1
        fi
    else
        echo ""
    fi

    return 0
}

# 1. PostgreSQL
echo -n "[1/$TOTAL_CHECKS] Checking PostgreSQL... "
if psql $DATABASE_URL -c "SELECT 1;" > /dev/null 2>&1; then
    echo "✅"
    ((CHECKS_PASSED++))
else
    echo "❌ (connection failed)"
fi

# 2. Redis
echo -n "[2/$TOTAL_CHECKS] Checking Redis... "
if redis-cli PING > /dev/null 2>&1; then
    echo "✅"
    ((CHECKS_PASSED++))
else
    echo "❌ (not responding)"
fi

# 3. Vault (optional)
echo -n "[3/$TOTAL_CHECKS] Checking Vault... "
if command -v vault > /dev/null 2>&1 && vault status > /dev/null 2>&1; then
    echo "✅"
    ((CHECKS_PASSED++))
else
    echo "⚠️ (not installed or not running)"
fi

# 4. Trading Service
if check_service "Trading Service" 50051 "http://localhost:8080/health"; then
    ((CHECKS_PASSED++))
fi

# 5. Backtesting Service
if check_service "Backtesting Service" 50052 "http://localhost:8082/health"; then
    ((CHECKS_PASSED++))
fi

# 6. ML Training Service
if check_service "ML Training Service" 50053 "http://localhost:8083/health"; then
    ((CHECKS_PASSED++))
fi

# 7. Prometheus (optional)
echo -n "[7/$TOTAL_CHECKS] Checking Prometheus... "
if curl -s -f http://localhost:9090/-/ready > /dev/null 2>&1; then
    echo "✅"
    ((CHECKS_PASSED++))
else
    echo "⚠️ (not running or not accessible)"
fi

# 8. Grafana (optional)
echo -n "[8/$TOTAL_CHECKS] Checking Grafana... "
if curl -s -f http://localhost:3000/api/health > /dev/null 2>&1; then
    echo "✅"
    ((CHECKS_PASSED++))
else
    echo "⚠️ (not running or not accessible)"
fi

# 9. Disk Space
echo -n "[9/$TOTAL_CHECKS] Checking Disk Space... "
DISK_USAGE=$(df -h / | awk 'NR==2 {print $5}' | sed 's/%//')
if [ "$DISK_USAGE" -lt 90 ]; then
    echo "✅ (${DISK_USAGE}% used)"
    ((CHECKS_PASSED++))
else
    echo "⚠️ (${DISK_USAGE}% used - WARNING)"
fi

# 10. Memory Usage
echo -n "[10/$TOTAL_CHECKS] Checking Memory Usage... "
MEM_USAGE=$(free | awk 'NR==2 {printf "%.0f", $3*100/$2}')
if [ "$MEM_USAGE" -lt 90 ]; then
    echo "✅ (${MEM_USAGE}% used)"
    ((CHECKS_PASSED++))
else
    echo "⚠️ (${MEM_USAGE}% used - WARNING)"
fi

# 11. CPU Load
echo -n "[11/$TOTAL_CHECKS] Checking CPU Load... "
CPU_LOAD=$(uptime | awk -F'load average:' '{print $2}' | awk '{print $1}' | sed 's/,//')
CPU_CORES=$(nproc)
if (( $(echo "$CPU_LOAD < $CPU_CORES" | bc -l) )); then
    echo "✅ (load: $CPU_LOAD, cores: $CPU_CORES)"
    ((CHECKS_PASSED++))
else
    echo "⚠️ (load: $CPU_LOAD, cores: $CPU_CORES - HIGH LOAD)"
fi

# 12. Time Synchronization
echo -n "[12/$TOTAL_CHECKS] Checking Time Sync... "
if command -v chronyc > /dev/null 2>&1; then
    TIME_OFFSET=$(chronyc tracking 2>/dev/null | grep 'System time' | awk '{print $4}' || echo "1.0")
    if (( $(echo "$TIME_OFFSET < 0.001" | bc -l) )); then
        echo "✅ (offset: ${TIME_OFFSET}s)"
        ((CHECKS_PASSED++))
    else
        echo "⚠️ (offset: ${TIME_OFFSET}s - WARNING)"
    fi
else
    echo "⚠️ (chrony not installed)"
fi

echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "  HEALTH CHECK SUMMARY: $CHECKS_PASSED/$TOTAL_CHECKS PASSED"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

if [ "$CHECKS_PASSED" -eq "$TOTAL_CHECKS" ]; then
    echo "  ✅ ALL CHECKS PASSED - SYSTEM HEALTHY"
    exit 0
elif [ "$CHECKS_PASSED" -ge $((TOTAL_CHECKS * 3 / 4)) ]; then
    echo "  ⚠️  MOST CHECKS PASSED - REVIEW WARNINGS"
    exit 1
else
    echo "  ❌ CRITICAL FAILURES - IMMEDIATE ACTION REQUIRED"
    exit 2
fi