#!/bin/bash
# Comprehensive Health Check Validation Script
# Tests all services including MLTrainingService for production readiness
#
# Based on production deployment requirements and expert analysis

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# Service endpoints and health check paths
declare -A SERVICES=(
    ["trading-service"]="http://localhost:8080/health"
    ["risk-management"]="http://localhost:8081/health"
    ["ml-training-service"]="http://localhost:8082/health"
    ["market-data"]="http://localhost:8083/health"
    ["tli-dashboard"]="http://localhost:8084/health"
)

# Timeout for health checks (seconds)
TIMEOUT=10
FAILED_SERVICES=0
TOTAL_SERVICES=${#SERVICES[@]}

log() {
    echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"
}

success() {
    echo -e "${GREEN}[✓]${NC} $1"
}

error() {
    echo -e "${RED}[✗]${NC} $1"
    ((FAILED_SERVICES++))
}

warning() {
    echo -e "${YELLOW}[⚠]${NC} $1"
}

echo "=========================================="
echo "  Foxhunt HFT Health Check Validation"
echo "=========================================="
echo

# Check if curl is available
if ! command -v curl &> /dev/null; then
    error "curl is required but not installed"
    exit 1
fi

# Check if jq is available for JSON parsing
if ! command -v jq &> /dev/null; then
    warning "jq not available - JSON response parsing will be limited"
    JQ_AVAILABLE=false
else
    JQ_AVAILABLE=true
fi

# =============================================================================
# SERVICE HEALTH CHECKS
# =============================================================================

log "Starting health check validation for all services..."
echo

for service in "${!SERVICES[@]}"; do
    endpoint="${SERVICES[$service]}"
    log "Checking health of $service at $endpoint"
    
    # Perform health check with timeout
    if response=$(curl -s --max-time $TIMEOUT "$endpoint" 2>/dev/null); then
        # Check if response contains health indicators
        if echo "$response" | grep -qi "healthy\|ok\|running\|up"; then
            success "$service is healthy"
            
            # Parse detailed health info if JSON and jq available
            if $JQ_AVAILABLE && echo "$response" | jq . >/dev/null 2>&1; then
                # Extract key health metrics
                if status=$(echo "$response" | jq -r '.status // .health // "unknown"' 2>/dev/null); then
                    echo "   Status: $status"
                fi
                
                if uptime=$(echo "$response" | jq -r '.uptime // "unknown"' 2>/dev/null); then
                    echo "   Uptime: $uptime"
                fi
                
                if version=$(echo "$response" | jq -r '.version // "unknown"' 2>/dev/null); then
                    echo "   Version: $version"
                fi
                
                # Service-specific health checks
                case $service in
                    "ml-training-service")
                        if model_status=$(echo "$response" | jq -r '.models.status // "unknown"' 2>/dev/null); then
                            echo "   ML Models: $model_status"
                        fi
                        if gpu_available=$(echo "$response" | jq -r '.gpu.available // "unknown"' 2>/dev/null); then
                            echo "   GPU Available: $gpu_available"
                        fi
                        ;;
                    "risk-management")
                        if position_count=$(echo "$response" | jq -r '.positions.count // "unknown"' 2>/dev/null); then
                            echo "   Active Positions: $position_count"
                        fi
                        if risk_limits=$(echo "$response" | jq -r '.risk_limits.status // "unknown"' 2>/dev/null); then
                            echo "   Risk Limits: $risk_limits"
                        fi
                        ;;
                    "trading-service")
                        if order_queue=$(echo "$response" | jq -r '.orders.queue_size // "unknown"' 2>/dev/null); then
                            echo "   Order Queue: $order_queue"
                        fi
                        if latency=$(echo "$response" | jq -r '.performance.avg_latency_us // "unknown"' 2>/dev/null); then
                            echo "   Average Latency: ${latency}μs"
                        fi
                        ;;
                esac
            fi
        else
            error "$service returned unhealthy status: $response"
        fi
    else
        error "$service health check failed - service may be down or unreachable"
        echo "   Endpoint: $endpoint"
        echo "   Check if service is running and endpoint is correct"
    fi
    echo
done

# =============================================================================
# ML TRAINING SERVICE SPECIFIC VALIDATION
# =============================================================================

log "Performing ML Training Service specific validation..."

# Check ML service configuration
ML_CONFIG_PATH="services/ml_training_service/config"
if [[ -d "$ML_CONFIG_PATH" ]]; then
    success "ML Training Service configuration directory found"
    
    # Check for required config files
    if [[ -f "$ML_CONFIG_PATH/training.toml" ]] || [[ -f "$ML_CONFIG_PATH/models.toml" ]]; then
        success "ML Training Service configuration files present"
    else
        warning "ML Training Service configuration files missing"
    fi
else
    warning "ML Training Service configuration directory not found"
fi

# Check ML model storage
ML_MODELS_PATH="models"
if [[ -d "$ML_MODELS_PATH" ]]; then
    model_count=$(find "$ML_MODELS_PATH" -name "*.onnx" -o -name "*.pt" -o -name "*.safetensors" | wc -l)
    if [[ $model_count -gt 0 ]]; then
        success "Found $model_count ML model files"
    else
        warning "No ML model files found in $ML_MODELS_PATH"
    fi
else
    warning "ML models directory not found"
fi

# =============================================================================
# DATABASE CONNECTIVITY CHECKS
# =============================================================================

log "Checking database connectivity..."

# PostgreSQL (primary database)
if command -v psql &> /dev/null; then
    if PGPASSWORD="${DB_PASSWORD:-foxhunt}" psql -h "${DB_HOST:-localhost}" -U "${DB_USER:-foxhunt}" -d "${DB_NAME:-foxhunt}" -c "SELECT 1;" >/dev/null 2>&1; then
        success "PostgreSQL database connection healthy"
    else
        error "PostgreSQL database connection failed"
    fi
else
    warning "psql not available - cannot test PostgreSQL connection"
fi

# Redis (caching)
if command -v redis-cli &> /dev/null; then
    if redis-cli -h "${REDIS_HOST:-localhost}" -p "${REDIS_PORT:-6379}" ping | grep -q "PONG"; then
        success "Redis connection healthy"
    else
        error "Redis connection failed"
    fi
else
    warning "redis-cli not available - cannot test Redis connection"
fi

# InfluxDB (metrics)
if command -v influx &> /dev/null; then
    if influx ping >/dev/null 2>&1; then
        success "InfluxDB connection healthy"
    else
        error "InfluxDB connection failed"
    fi
else
    warning "influx CLI not available - cannot test InfluxDB connection"
fi

# =============================================================================
# MONITORING AND METRICS VALIDATION
# =============================================================================

log "Validating monitoring and metrics endpoints..."

# Prometheus metrics
PROMETHEUS_ENDPOINTS=(
    "http://localhost:8080/metrics"  # Trading Service
    "http://localhost:8081/metrics"  # Risk Management
    "http://localhost:8082/metrics"  # ML Training Service
)

for endpoint in "${PROMETHEUS_ENDPOINTS[@]}"; do
    service_name=$(echo "$endpoint" | sed 's/.*:\([0-9]*\).*/Port \1/')
    if curl -s --max-time 5 "$endpoint" | grep -q "^# HELP"; then
        success "Prometheus metrics available for $service_name"
    else
        warning "Prometheus metrics not available for $service_name"
    fi
done

# =============================================================================
# PERFORMANCE VALIDATION
# =============================================================================

log "Performing basic performance validation..."

# Check system resources
load_avg=$(uptime | awk -F'load average:' '{ print $2 }' | awk '{ print $1 }' | sed 's/,//')
if (( $(echo "$load_avg < 2.0" | bc -l) )); then
    success "System load average acceptable: $load_avg"
else
    warning "High system load average: $load_avg"
fi

# Check memory usage
if command -v free &> /dev/null; then
    mem_usage=$(free | grep Mem | awk '{printf "%.1f", $3/$2 * 100.0}')
    if (( $(echo "$mem_usage < 80.0" | bc -l) )); then
        success "Memory usage acceptable: ${mem_usage}%"
    else
        warning "High memory usage: ${mem_usage}%"
    fi
fi

# Check disk space
disk_usage=$(df / | tail -1 | awk '{print $5}' | sed 's/%//')
if [[ $disk_usage -lt 80 ]]; then
    success "Disk usage acceptable: ${disk_usage}%"
else
    warning "High disk usage: ${disk_usage}%"
fi

# =============================================================================
# SECURITY VALIDATION
# =============================================================================

log "Performing security validation..."

# Check for secure configuration
if [[ -f ".env" ]]; then
    if grep -q "PASSWORD.*changeme\|SECRET.*default\|KEY.*example" .env; then
        error "Default credentials detected in .env file"
    else
        success "No default credentials found in .env"
    fi
fi

# Check file permissions on sensitive files
if [[ -f ".env" ]]; then
    env_perms=$(stat -c "%a" .env 2>/dev/null || stat -f "%A" .env 2>/dev/null)
    if [[ "$env_perms" = "600" ]] || [[ "$env_perms" = "0600" ]]; then
        success ".env file has secure permissions"
    else
        error ".env file has insecure permissions: $env_perms"
    fi
fi

# =============================================================================
# FINAL SUMMARY
# =============================================================================

echo
echo "=========================================="
echo "       HEALTH CHECK SUMMARY"
echo "=========================================="
echo

HEALTHY_SERVICES=$((TOTAL_SERVICES - FAILED_SERVICES))

echo -e "Services Healthy: ${GREEN}$HEALTHY_SERVICES${NC}/$TOTAL_SERVICES"
echo -e "Services Failed:  ${RED}$FAILED_SERVICES${NC}/$TOTAL_SERVICES"
echo

if [[ $FAILED_SERVICES -eq 0 ]]; then
    echo -e "${GREEN}✅ ALL SERVICES HEALTHY${NC}"
    echo
    echo "🎉 System is ready for production operation!"
    echo
    echo "All services are responding correctly and health checks pass."
    exit 0
elif [[ $FAILED_SERVICES -lt $((TOTAL_SERVICES / 2)) ]]; then
    echo -e "${YELLOW}⚠️ PARTIAL SERVICE FAILURES${NC}"
    echo
    echo "🔧 Some services need attention before full production deployment."
    echo
    echo "Review failed service logs and restart as needed."
    exit 1
else
    echo -e "${RED}❌ CRITICAL SERVICE FAILURES${NC}"
    echo
    echo "🚨 Multiple service failures detected - deployment not recommended!"
    echo
    echo "Investigate and resolve service issues before proceeding."
    exit 2
fi