foxhunt/scripts/comprehensive_health_check.sh

#!/bin/bash

# Foxhunt HFT System - Comprehensive Health Check
# Enhanced version with dependency validation and detailed reporting

# Color codes for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Counters
TOTAL_CHECKS=0
PASSED_CHECKS=0
FAILED_CHECKS=0

# Function to check health
check_health() {
    local name=$1
    local command=$2

    TOTAL_CHECKS=$((TOTAL_CHECKS + 1))

    if eval "$command" > /dev/null 2>&1; then
        echo -e "  ${GREEN}✓${NC} $name: ${GREEN}OK${NC}"
        PASSED_CHECKS=$((PASSED_CHECKS + 1))
        return 0
    else
        echo -e "  ${RED}✗${NC} $name: ${RED}FAIL${NC}"
        FAILED_CHECKS=$((FAILED_CHECKS + 1))
        return 1
    fi
}

echo "================================================================================"
echo "                    Foxhunt Health Check - Comprehensive"
echo "================================================================================"
echo "Timestamp: $(date '+%Y-%m-%d %H:%M:%S')"
echo ""

# 1. Docker Container Status
echo -e "${BLUE}[1/7] Docker Container Status:${NC}"
check_health "Docker Compose" "docker-compose ps | grep -E '(healthy|Up)'"
docker-compose ps 2>/dev/null || echo "  (docker-compose not running)"
echo ""

# 2. Infrastructure Layer
echo -e "${BLUE}[2/7] Infrastructure Health:${NC}"
check_health "PostgreSQL" "docker exec foxhunt-postgres pg_isready -U foxhunt 2>/dev/null"
check_health "Redis" "docker exec foxhunt-redis redis-cli ping 2>/dev/null | grep -q PONG"
check_health "Vault" "curl -sf http://localhost:8200/v1/sys/health"
check_health "InfluxDB" "curl -sf http://localhost:8086/health"
echo ""

# 3. HTTP Health Endpoints
echo -e "${BLUE}[3/7] HTTP Health Endpoints:${NC}"
check_health "API Gateway (HTTP)" "curl -sf http://localhost:8080/health"
check_health "Trading Service (HTTP)" "curl -sf http://localhost:8081/health"
check_health "Backtesting Service (HTTP)" "curl -sf http://localhost:8083/health"
check_health "ML Training Service (HTTP)" "curl -sf http://localhost:8095/health"
echo ""

# 4. gRPC Services
echo -e "${BLUE}[4/7] gRPC Services:${NC}"
if command -v grpc_health_probe &> /dev/null; then
    check_health "API Gateway (gRPC)" "grpc_health_probe -addr=localhost:50051"
    check_health "Trading Service (gRPC)" "grpc_health_probe -addr=localhost:50052"
    check_health "Backtesting Service (gRPC)" "grpc_health_probe -addr=localhost:50053"
    check_health "ML Training Service (gRPC)" "grpc_health_probe -addr=localhost:50054"
else
    echo -e "  ${YELLOW}⚠${NC} grpc_health_probe not installed - skipping gRPC checks"
    echo "    Install: https://github.com/grpc-ecosystem/grpc-health-probe"
fi
echo ""

# 5. Database Connectivity
echo -e "${BLUE}[5/7] Database Connectivity:${NC}"
check_health "PostgreSQL Query" "docker exec foxhunt-postgres psql -U foxhunt -d foxhunt -c 'SELECT 1;' 2>/dev/null"
check_health "Redis Keyspace" "docker exec foxhunt-redis redis-cli INFO keyspace 2>/dev/null"

if command -v jq &> /dev/null; then
    check_health "InfluxDB Buckets" "curl -sf -H 'Authorization: Token foxhunt-dev-token' http://localhost:8086/api/v2/buckets | jq -e '.buckets | length > 0'"
else
    echo -e "  ${YELLOW}⚠${NC} jq not installed - skipping InfluxDB bucket check"
fi
echo ""

# 6. Monitoring Stack
echo -e "${BLUE}[6/7] Monitoring Stack:${NC}"
check_health "Prometheus" "curl -sf http://localhost:9090/-/healthy"
check_health "Grafana" "curl -sf http://localhost:3000/api/health"

if command -v jq &> /dev/null; then
    check_health "Prometheus Targets" "curl -sf http://localhost:9090/api/v1/targets | jq -e '.data.activeTargets | length > 0'"
else
    echo -e "  ${YELLOW}⚠${NC} jq not installed - skipping Prometheus target check"
fi
echo ""

# 7. Service Dependencies
echo -e "${BLUE}[7/7] Service Dependencies:${NC}"

if command -v jq &> /dev/null; then
    # API Gateway dependencies
    API_GW_HEALTH=$(curl -sf http://localhost:8080/health 2>/dev/null | jq -r '.dependencies.postgres' 2>/dev/null)
    if [ "$API_GW_HEALTH" = "healthy" ]; then
        echo -e "  ${GREEN}✓${NC} API Gateway → PostgreSQL: ${GREEN}OK${NC}"
        PASSED_CHECKS=$((PASSED_CHECKS + 1))
    else
        echo -e "  ${RED}✗${NC} API Gateway → PostgreSQL: ${RED}FAIL${NC}"
        FAILED_CHECKS=$((FAILED_CHECKS + 1))
    fi
    TOTAL_CHECKS=$((TOTAL_CHECKS + 1))

    API_GW_REDIS=$(curl -sf http://localhost:8080/health 2>/dev/null | jq -r '.dependencies.redis' 2>/dev/null)
    if [ "$API_GW_REDIS" = "healthy" ]; then
        echo -e "  ${GREEN}✓${NC} API Gateway → Redis: ${GREEN}OK${NC}"
        PASSED_CHECKS=$((PASSED_CHECKS + 1))
    else
        echo -e "  ${RED}✗${NC} API Gateway → Redis: ${RED}FAIL${NC}"
        FAILED_CHECKS=$((FAILED_CHECKS + 1))
    fi
    TOTAL_CHECKS=$((TOTAL_CHECKS + 1))
else
    echo -e "  ${YELLOW}⚠${NC} jq not installed - skipping dependency checks"
    echo "    Install: sudo apt-get install jq"
fi
echo ""

# Resource Usage
echo -e "${BLUE}Resource Usage:${NC}"
if command -v docker &> /dev/null; then
    echo "  Container Memory Usage:"
    docker stats --no-stream --format "    {{.Name}}: {{.MemUsage}}" 2>/dev/null | head -n 10 || echo "    (no containers running)"
fi
echo ""

# Summary
echo "================================================================================"
echo -e "${YELLOW}Health Check Summary${NC}"
echo "================================================================================"
echo "Total Checks: $TOTAL_CHECKS"
echo -e "Passed: ${GREEN}$PASSED_CHECKS${NC}"
echo -e "Failed: ${RED}$FAILED_CHECKS${NC}"

if [ $FAILED_CHECKS -eq 0 ]; then
    echo ""
    echo -e "${GREEN}✓ All health checks passed!${NC}"
    echo "System is ready for operation."
    exit 0
else
    echo ""
    echo -e "${RED}✗ Some health checks failed!${NC}"
    echo "Please review failed checks above and consult troubleshooting documentation."
    exit 1
fi