foxhunt/health_check.sh

#!/bin/bash
# Foxhunt HFT System - Comprehensive Health Check Script
# Wave 75 Agent 6 - Service Health Validation
#
# This script validates:
# - 4 gRPC Application Services (ports 50050-50053)
# - 6 Infrastructure Services (PostgreSQL, Redis, Vault, InfluxDB, Prometheus, Grafana)
# - Inter-service communication
# - Resource usage
# - Hot-reload functionality

set -e

# Color codes for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Results tracking
TOTAL_CHECKS=0
PASSED_CHECKS=0
FAILED_CHECKS=0
WARNING_CHECKS=0

# Logging
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
LOG_DIR="./logs"
HEALTH_LOG="${LOG_DIR}/health_check_${TIMESTAMP}.log"

mkdir -p "$LOG_DIR"

# Logging functions
log_info() {
    echo -e "${BLUE}[INFO]${NC} $1" | tee -a "$HEALTH_LOG"
}

log_success() {
    echo -e "${GREEN}[PASS]${NC} $1" | tee -a "$HEALTH_LOG"
    ((PASSED_CHECKS++))
}

log_error() {
    echo -e "${RED}[FAIL]${NC} $1" | tee -a "$HEALTH_LOG"
    ((FAILED_CHECKS++))
}

log_warning() {
    echo -e "${YELLOW}[WARN]${NC} $1" | tee -a "$HEALTH_LOG"
    ((WARNING_CHECKS++))
}

log_header() {
    echo -e "\n${BLUE}========================================${NC}" | tee -a "$HEALTH_LOG"
    echo -e "${BLUE}$1${NC}" | tee -a "$HEALTH_LOG"
    echo -e "${BLUE}========================================${NC}\n" | tee -a "$HEALTH_LOG"
}

# Check if required tools are installed
check_prerequisites() {
    log_header "Checking Prerequisites"

    local tools=("grpcurl" "psql" "curl" "jq" "docker")
    local missing_tools=()

    # redis-cli is optional (can use docker exec)

    for tool in "${tools[@]}"; do
        ((TOTAL_CHECKS++))
        if command -v "$tool" &> /dev/null; then
            log_success "$tool is installed"
        else
            log_error "$tool is NOT installed"
            missing_tools+=("$tool")
        fi
    done

    if [ ${#missing_tools[@]} -gt 0 ]; then
        log_error "Missing required tools: ${missing_tools[*]}"
        log_info "Install missing tools before proceeding"
        return 1
    fi

    return 0
}

# Check gRPC service health
check_grpc_service() {
    local service_name=$1
    local port=$2
    local package=$3
    local service=$4

    log_info "Checking $service_name on port $port..."
    ((TOTAL_CHECKS++))

    # Check if port is listening
    if ! netstat -tuln 2>/dev/null | grep -q ":$port "; then
        if ! ss -tuln 2>/dev/null | grep -q ":$port "; then
            log_error "$service_name: Port $port is NOT listening"
            return 1
        fi
    fi
    log_success "$service_name: Port $port is listening"

    # List available services
    ((TOTAL_CHECKS++))
    if grpcurl -plaintext localhost:$port list > /dev/null 2>&1; then
        log_success "$service_name: gRPC server responding"

        # Get service list
        local services=$(grpcurl -plaintext localhost:$port list 2>/dev/null)
        echo "  Available services:" >> "$HEALTH_LOG"
        echo "$services" | sed 's/^/    /' >> "$HEALTH_LOG"
    else
        log_error "$service_name: gRPC server NOT responding"
        return 1
    fi

    # Check health endpoint
    ((TOTAL_CHECKS++))
    if grpcurl -plaintext localhost:$port grpc.health.v1.Health/Check 2>&1 | grep -q "SERVING"; then
        log_success "$service_name: Health check SERVING"
    else
        log_warning "$service_name: Health check returned non-SERVING status or not implemented"
    fi

    return 0
}

# Check infrastructure service
check_infrastructure_service() {
    local service_name=$1
    local check_command=$2

    log_info "Checking $service_name..."
    ((TOTAL_CHECKS++))

    if eval "$check_command" > /dev/null 2>&1; then
        log_success "$service_name is healthy"
        return 0
    else
        log_error "$service_name is NOT healthy"
        return 1
    fi
}

# Check gRPC application services
check_grpc_services() {
    log_header "Checking gRPC Application Services"

    # API Gateway (port 50050)
    check_grpc_service "API Gateway" 50050 "foxhunt" "ApiGateway"

    # Trading Service (port 50051)
    check_grpc_service "Trading Service" 50051 "trading" "TradingService"

    # Backtesting Service (port 50052)
    check_grpc_service "Backtesting Service" 50052 "backtesting" "BacktestingService"

    # ML Training Service (port 50053)
    check_grpc_service "ML Training Service" 50053 "ml_training" "MLTrainingService"
}

# Check infrastructure services
check_infrastructure_services() {
    log_header "Checking Infrastructure Services"

    # PostgreSQL (port 5433)
    log_info "Checking PostgreSQL on port 5433..."
    ((TOTAL_CHECKS++))
    # Try test credentials first (from docker-compose)
    if PGPASSWORD=test_password psql -h localhost -p 5433 -U foxhunt_test -d foxhunt_test -c "SELECT 1;" > /dev/null 2>&1; then
        log_success "PostgreSQL is healthy (test database)"

        # Check database exists and has tables
        ((TOTAL_CHECKS++))
        local table_count=$(PGPASSWORD=test_password psql -h localhost -p 5433 -U foxhunt_test -d foxhunt_test -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public';" 2>/dev/null | tr -d ' ')
        if [ "$table_count" -gt 0 ]; then
            log_success "PostgreSQL has $table_count tables"
        else
            log_warning "PostgreSQL database exists but has no tables"
        fi
    elif PGPASSWORD=postgres psql -h localhost -p 5433 -U postgres -d foxhunt -c "SELECT 1;" > /dev/null 2>&1; then
        log_success "PostgreSQL is healthy (production database)"

        # Check database exists and has tables
        ((TOTAL_CHECKS++))
        local table_count=$(PGPASSWORD=postgres psql -h localhost -p 5433 -U postgres -d foxhunt -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public';" 2>/dev/null | tr -d ' ')
        if [ "$table_count" -gt 0 ]; then
            log_success "PostgreSQL has $table_count tables"
        else
            log_warning "PostgreSQL database exists but has no tables"
        fi
    else
        log_error "PostgreSQL is NOT healthy (tried both test and production credentials)"
    fi

    # Redis (port 6380)
    log_info "Checking Redis on port 6380..."
    ((TOTAL_CHECKS++))
    # Try native redis-cli first, fallback to docker
    if command -v redis-cli &> /dev/null && redis-cli -p 6380 PING 2>&1 | grep -q "PONG"; then
        log_success "Redis is healthy (native client)"

        # Check Redis memory usage
        ((TOTAL_CHECKS++))
        local redis_memory=$(redis-cli -p 6380 INFO memory 2>/dev/null | grep "used_memory_human" | cut -d':' -f2 | tr -d '\r')
        if [ -n "$redis_memory" ]; then
            log_success "Redis memory usage: $redis_memory"
        fi
    elif docker exec api_gateway_test_redis redis-cli PING 2>&1 | grep -q "PONG"; then
        log_success "Redis is healthy (via Docker)"

        # Check Redis memory usage
        ((TOTAL_CHECKS++))
        local redis_memory=$(docker exec api_gateway_test_redis redis-cli INFO memory 2>/dev/null | grep "used_memory_human" | cut -d':' -f2 | tr -d '\r')
        if [ -n "$redis_memory" ]; then
            log_success "Redis memory usage: $redis_memory"
        fi
    else
        log_error "Redis is NOT healthy"
    fi

    # Vault (port 8200)
    log_info "Checking Vault on port 8200..."
    ((TOTAL_CHECKS++))
    local vault_health=$(curl -s http://localhost:8200/v1/sys/health 2>/dev/null)
    if [ -n "$vault_health" ]; then
        local vault_sealed=$(echo "$vault_health" | jq -r '.sealed' 2>/dev/null)
        if [ "$vault_sealed" == "false" ]; then
            log_success "Vault is healthy and unsealed"
        elif [ "$vault_sealed" == "true" ]; then
            log_warning "Vault is healthy but SEALED"
        else
            log_success "Vault is responding"
        fi
    else
        log_error "Vault is NOT responding"
    fi

    # InfluxDB (port 8086) - Not running based on docker ps
    log_info "Checking InfluxDB on port 8086..."
    ((TOTAL_CHECKS++))
    if curl -s http://localhost:8086/health > /dev/null 2>&1; then
        log_success "InfluxDB is healthy"
    else
        log_warning "InfluxDB is NOT running (optional service)"
    fi

    # Prometheus (port 9099 mapped to 9090)
    log_info "Checking Prometheus on port 9099..."
    ((TOTAL_CHECKS++))
    if curl -s http://localhost:9099/-/healthy 2>&1 | grep -q "Prometheus"; then
        log_success "Prometheus is healthy"
    else
        log_error "Prometheus is NOT healthy"
    fi

    # Grafana (port 3000)
    log_info "Checking Grafana on port 3000..."
    ((TOTAL_CHECKS++))
    local grafana_health=$(curl -s http://localhost:3000/api/health 2>/dev/null)
    if echo "$grafana_health" | jq -e '.database == "ok"' > /dev/null 2>&1; then
        log_success "Grafana is healthy"
    else
        log_warning "Grafana is responding but may have issues"
    fi
}

# Check Docker containers
check_docker_containers() {
    log_header "Checking Docker Containers"

    log_info "Running Docker containers:"
    docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" | tee -a "$HEALTH_LOG"

    # Check for unhealthy containers
    ((TOTAL_CHECKS++))
    local unhealthy=$(docker ps --filter "health=unhealthy" --format "{{.Names}}" 2>/dev/null)
    if [ -z "$unhealthy" ]; then
        log_success "No unhealthy containers detected"
    else
        log_error "Unhealthy containers detected: $unhealthy"
    fi
}

# Check service processes
check_service_processes() {
    log_header "Checking Service Processes"

    local services=("trading_service" "backtesting_service" "ml_training_service" "api_gateway")

    for service in "${services[@]}"; do
        ((TOTAL_CHECKS++))
        if pgrep -f "$service" > /dev/null; then
            local pid=$(pgrep -f "$service")
            local mem_usage=$(ps -p $pid -o %mem --no-headers 2>/dev/null | tr -d ' ')
            local cpu_usage=$(ps -p $pid -o %cpu --no-headers 2>/dev/null | tr -d ' ')
            log_success "$service is running (PID: $pid, CPU: ${cpu_usage}%, MEM: ${mem_usage}%)"
        else
            log_error "$service is NOT running"
        fi
    done
}

# Check resource usage
check_resource_usage() {
    log_header "Checking System Resource Usage"

    # CPU usage
    ((TOTAL_CHECKS++))
    local cpu_usage=$(top -bn1 | grep "Cpu(s)" | sed "s/.*, *\([0-9.]*\)%* id.*/\1/" | awk '{print 100 - $1}')
    if (( $(echo "$cpu_usage < 80" | bc -l) )); then
        log_success "CPU usage: ${cpu_usage}% (healthy)"
    else
        log_warning "CPU usage: ${cpu_usage}% (high)"
    fi

    # Memory usage
    ((TOTAL_CHECKS++))
    local mem_total=$(free -g | awk '/^Mem:/{print $2}')
    local mem_used=$(free -g | awk '/^Mem:/{print $3}')
    local mem_percent=$(awk "BEGIN {printf \"%.1f\", ($mem_used/$mem_total)*100}")

    if (( $(echo "$mem_percent < 80" | bc -l) )); then
        log_success "Memory usage: ${mem_used}GB/${mem_total}GB (${mem_percent}%) (healthy)"
    else
        log_warning "Memory usage: ${mem_used}GB/${mem_total}GB (${mem_percent}%) (high)"
    fi

    # Disk usage
    ((TOTAL_CHECKS++))
    local disk_usage=$(df -h . | awk 'NR==2 {print $5}' | sed 's/%//')
    if [ "$disk_usage" -lt 80 ]; then
        log_success "Disk usage: ${disk_usage}% (healthy)"
    else
        log_warning "Disk usage: ${disk_usage}% (high)"
    fi
}

# Test inter-service communication
test_inter_service_communication() {
    log_header "Testing Inter-Service Communication"

    log_info "Testing API Gateway routing to Trading Service..."
    ((TOTAL_CHECKS++))

    # Check if we can list services through API Gateway
    if grpcurl -plaintext localhost:50050 list 2>&1 | grep -q "trading.TradingService"; then
        log_success "API Gateway can see Trading Service"
    else
        log_warning "API Gateway may not have Trading Service registered"
    fi
}

# Test hot-reload functionality
test_hot_reload() {
    log_header "Testing Hot-Reload Functionality"

    log_info "Checking PostgreSQL NOTIFY/LISTEN support..."
    ((TOTAL_CHECKS++))

    # Check if config_settings table exists (try both databases)
    if PGPASSWORD=test_password psql -h localhost -p 5433 -U foxhunt_test -d foxhunt_test -c "\d config_settings" > /dev/null 2>&1; then
        log_success "config_settings table exists (test database)"

        # Count configuration entries
        ((TOTAL_CHECKS++))
        local config_count=$(PGPASSWORD=test_password psql -h localhost -p 5433 -U foxhunt_test -d foxhunt_test -t -c "SELECT COUNT(*) FROM config_settings;" 2>/dev/null | tr -d ' ')
        if [ "$config_count" -gt 0 ]; then
            log_success "Found $config_count configuration entries"
        else
            log_warning "config_settings table exists but is empty"
        fi
    elif PGPASSWORD=postgres psql -h localhost -p 5433 -U postgres -d foxhunt -c "\d config_settings" > /dev/null 2>&1; then
        log_success "config_settings table exists (production database)"

        # Count configuration entries
        ((TOTAL_CHECKS++))
        local config_count=$(PGPASSWORD=postgres psql -h localhost -p 5433 -U postgres -d foxhunt -t -c "SELECT COUNT(*) FROM config_settings;" 2>/dev/null | tr -d ' ')
        if [ "$config_count" -gt 0 ]; then
            log_success "Found $config_count configuration entries"
        else
            log_warning "config_settings table exists but is empty"
        fi
    else
        log_warning "config_settings table does not exist (hot-reload may not be configured)"
    fi
}

# Check service logs for errors
check_service_logs() {
    log_header "Checking Service Logs for Recent Errors"

    local log_files=("api_gateway.log" "trading_service.log" "backtesting_service.log" "ml_training_service.log")

    for log_file in "${log_files[@]}"; do
        local log_path="${LOG_DIR}/${log_file}"
        if [ -f "$log_path" ]; then
            ((TOTAL_CHECKS++))
            local error_count=$(grep -i "error\|panic\|fatal" "$log_path" 2>/dev/null | wc -l)

            if [ "$error_count" -eq 0 ]; then
                log_success "$log_file: No errors detected"
            elif [ "$error_count" -lt 5 ]; then
                log_warning "$log_file: $error_count errors detected (review recommended)"
            else
                log_error "$log_file: $error_count errors detected (attention required)"
            fi
        else
            log_warning "$log_file: Log file not found"
        fi
    done
}

# Generate summary report
generate_summary() {
    log_header "Health Check Summary"

    echo "" | tee -a "$HEALTH_LOG"
    echo "Total Checks:   $TOTAL_CHECKS" | tee -a "$HEALTH_LOG"
    echo -e "${GREEN}Passed:         $PASSED_CHECKS${NC}" | tee -a "$HEALTH_LOG"
    echo -e "${YELLOW}Warnings:       $WARNING_CHECKS${NC}" | tee -a "$HEALTH_LOG"
    echo -e "${RED}Failed:         $FAILED_CHECKS${NC}" | tee -a "$HEALTH_LOG"
    echo "" | tee -a "$HEALTH_LOG"

    local success_rate=$(awk "BEGIN {printf \"%.1f\", ($PASSED_CHECKS/$TOTAL_CHECKS)*100}")
    echo "Success Rate: ${success_rate}%" | tee -a "$HEALTH_LOG"

    if [ "$FAILED_CHECKS" -eq 0 ]; then
        echo -e "${GREEN}Overall Status: HEALTHY${NC}" | tee -a "$HEALTH_LOG"
        return 0
    elif [ "$FAILED_CHECKS" -lt 5 ]; then
        echo -e "${YELLOW}Overall Status: DEGRADED${NC}" | tee -a "$HEALTH_LOG"
        return 1
    else
        echo -e "${RED}Overall Status: UNHEALTHY${NC}" | tee -a "$HEALTH_LOG"
        return 2
    fi
}

# Main execution
main() {
    log_header "Foxhunt HFT System - Comprehensive Health Check"
    log_info "Starting health check at $(date)"
    log_info "Log file: $HEALTH_LOG"
    echo ""

    # Run all checks
    check_prerequisites || exit 1
    check_docker_containers
    check_infrastructure_services
    check_grpc_services
    check_service_processes
    check_resource_usage
    test_inter_service_communication
    test_hot_reload
    check_service_logs

    # Generate summary
    echo ""
    generate_summary

    log_info "Health check completed at $(date)"
    log_info "Detailed log saved to: $HEALTH_LOG"
}

# Run main function
main

exit $?