#!/bin/bash # Foxhunt HFT System - Comprehensive Health Check # Enhanced version with dependency validation and detailed reporting # Color codes for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color # Counters TOTAL_CHECKS=0 PASSED_CHECKS=0 FAILED_CHECKS=0 # Function to check health check_health() { local name=$1 local command=$2 TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) if eval "$command" > /dev/null 2>&1; then echo -e " ${GREEN}✓${NC} $name: ${GREEN}OK${NC}" PASSED_CHECKS=$((PASSED_CHECKS + 1)) return 0 else echo -e " ${RED}✗${NC} $name: ${RED}FAIL${NC}" FAILED_CHECKS=$((FAILED_CHECKS + 1)) return 1 fi } echo "================================================================================" echo " Foxhunt Health Check - Comprehensive" echo "================================================================================" echo "Timestamp: $(date '+%Y-%m-%d %H:%M:%S')" echo "" # 1. Docker Container Status echo -e "${BLUE}[1/7] Docker Container Status:${NC}" check_health "Docker Compose" "docker-compose ps | grep -E '(healthy|Up)'" docker-compose ps 2>/dev/null || echo " (docker-compose not running)" echo "" # 2. Infrastructure Layer echo -e "${BLUE}[2/7] Infrastructure Health:${NC}" check_health "PostgreSQL" "docker exec foxhunt-postgres pg_isready -U foxhunt 2>/dev/null" check_health "Redis" "docker exec foxhunt-redis redis-cli ping 2>/dev/null | grep -q PONG" check_health "Vault" "curl -sf http://localhost:8200/v1/sys/health" check_health "InfluxDB" "curl -sf http://localhost:8086/health" echo "" # 3. HTTP Health Endpoints echo -e "${BLUE}[3/7] HTTP Health Endpoints:${NC}" check_health "API Gateway (HTTP)" "curl -sf http://localhost:8080/health" check_health "Trading Service (HTTP)" "curl -sf http://localhost:8081/health" check_health "Backtesting Service (HTTP)" "curl -sf http://localhost:8083/health" check_health "ML Training Service (HTTP)" "curl -sf http://localhost:8095/health" echo "" # 4. gRPC Services echo -e "${BLUE}[4/7] gRPC Services:${NC}" if command -v grpc_health_probe &> /dev/null; then check_health "API Gateway (gRPC)" "grpc_health_probe -addr=localhost:50051" check_health "Trading Service (gRPC)" "grpc_health_probe -addr=localhost:50052" check_health "Backtesting Service (gRPC)" "grpc_health_probe -addr=localhost:50053" check_health "ML Training Service (gRPC)" "grpc_health_probe -addr=localhost:50054" else echo -e " ${YELLOW}⚠${NC} grpc_health_probe not installed - skipping gRPC checks" echo " Install: https://github.com/grpc-ecosystem/grpc-health-probe" fi echo "" # 5. Database Connectivity echo -e "${BLUE}[5/7] Database Connectivity:${NC}" check_health "PostgreSQL Query" "docker exec foxhunt-postgres psql -U foxhunt -d foxhunt -c 'SELECT 1;' 2>/dev/null" check_health "Redis Keyspace" "docker exec foxhunt-redis redis-cli INFO keyspace 2>/dev/null" if command -v jq &> /dev/null; then check_health "InfluxDB Buckets" "curl -sf -H 'Authorization: Token foxhunt-dev-token' http://localhost:8086/api/v2/buckets | jq -e '.buckets | length > 0'" else echo -e " ${YELLOW}⚠${NC} jq not installed - skipping InfluxDB bucket check" fi echo "" # 6. Monitoring Stack echo -e "${BLUE}[6/7] Monitoring Stack:${NC}" check_health "Prometheus" "curl -sf http://localhost:9090/-/healthy" check_health "Grafana" "curl -sf http://localhost:3000/api/health" if command -v jq &> /dev/null; then check_health "Prometheus Targets" "curl -sf http://localhost:9090/api/v1/targets | jq -e '.data.activeTargets | length > 0'" else echo -e " ${YELLOW}⚠${NC} jq not installed - skipping Prometheus target check" fi echo "" # 7. Service Dependencies echo -e "${BLUE}[7/7] Service Dependencies:${NC}" if command -v jq &> /dev/null; then # API Gateway dependencies API_GW_HEALTH=$(curl -sf http://localhost:8080/health 2>/dev/null | jq -r '.dependencies.postgres' 2>/dev/null) if [ "$API_GW_HEALTH" = "healthy" ]; then echo -e " ${GREEN}✓${NC} API Gateway → PostgreSQL: ${GREEN}OK${NC}" PASSED_CHECKS=$((PASSED_CHECKS + 1)) else echo -e " ${RED}✗${NC} API Gateway → PostgreSQL: ${RED}FAIL${NC}" FAILED_CHECKS=$((FAILED_CHECKS + 1)) fi TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) API_GW_REDIS=$(curl -sf http://localhost:8080/health 2>/dev/null | jq -r '.dependencies.redis' 2>/dev/null) if [ "$API_GW_REDIS" = "healthy" ]; then echo -e " ${GREEN}✓${NC} API Gateway → Redis: ${GREEN}OK${NC}" PASSED_CHECKS=$((PASSED_CHECKS + 1)) else echo -e " ${RED}✗${NC} API Gateway → Redis: ${RED}FAIL${NC}" FAILED_CHECKS=$((FAILED_CHECKS + 1)) fi TOTAL_CHECKS=$((TOTAL_CHECKS + 1)) else echo -e " ${YELLOW}⚠${NC} jq not installed - skipping dependency checks" echo " Install: sudo apt-get install jq" fi echo "" # Resource Usage echo -e "${BLUE}Resource Usage:${NC}" if command -v docker &> /dev/null; then echo " Container Memory Usage:" docker stats --no-stream --format " {{.Name}}: {{.MemUsage}}" 2>/dev/null | head -n 10 || echo " (no containers running)" fi echo "" # Summary echo "================================================================================" echo -e "${YELLOW}Health Check Summary${NC}" echo "================================================================================" echo "Total Checks: $TOTAL_CHECKS" echo -e "Passed: ${GREEN}$PASSED_CHECKS${NC}" echo -e "Failed: ${RED}$FAILED_CHECKS${NC}" if [ $FAILED_CHECKS -eq 0 ]; then echo "" echo -e "${GREEN}✓ All health checks passed!${NC}" echo "System is ready for operation." exit 0 else echo "" echo -e "${RED}✗ Some health checks failed!${NC}" echo "Please review failed checks above and consult troubleshooting documentation." exit 1 fi