foxhunt/deployment/scripts/pre-deployment-validation.sh

#!/bin/bash
# Pre-Deployment Validation Script for Foxhunt HFT Trading System
# Comprehensive pre-flight checks for zero-downtime deployment
#
# This script validates all dependencies, configurations, and system readiness
# before initiating a production deployment.

set -euo pipefail

# Configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
FOXHUNT_HOME="/opt/foxhunt"
CONFIG_DIR="${FOXHUNT_HOME}/config"
VALIDATION_LOG="/home/jgrusewski/Work/foxhunt/logs/pre-deployment-$(date +%s).log"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# Validation counters
CRITICAL_ISSUES=0
HIGH_ISSUES=0
MEDIUM_ISSUES=0
WARNINGS=0

# Create log directory if it doesn't exist
mkdir -p "$(dirname "$VALIDATION_LOG")"

log() {
    echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1" | tee -a "$VALIDATION_LOG"
}

error() {
    echo -e "${RED}[ERROR]${NC} $1" | tee -a "$VALIDATION_LOG"
    ((CRITICAL_ISSUES++))
}

warning() {
    echo -e "${YELLOW}[WARNING]${NC} $1" | tee -a "$VALIDATION_LOG"
    ((WARNINGS++))
}

success() {
    echo -e "${GREEN}[SUCCESS]${NC} $1" | tee -a "$VALIDATION_LOG"
}

critical() {
    echo -e "${RED}[CRITICAL]${NC} $1" | tee -a "$VALIDATION_LOG"
    ((CRITICAL_ISSUES++))
}

high() {
    echo -e "${YELLOW}[HIGH]${NC} $1" | tee -a "$VALIDATION_LOG"
    ((HIGH_ISSUES++))
}

# Print banner
echo "============================================================"
echo "  Foxhunt HFT Pre-Deployment Validation"
echo "============================================================"
echo

# =============================================================================
# SYSTEM REQUIREMENTS VALIDATION
# =============================================================================

log "Validating system requirements..."

# Check CPU architecture and features
if grep -q "avx2" /proc/cpuinfo; then
    success "AVX2 instruction set available for SIMD optimizations"
else
    critical "AVX2 instruction set not available - HFT performance will be degraded"
fi

if grep -q "rdtsc" /proc/cpuinfo; then
    success "RDTSC instruction available for nanosecond timing"
else
    critical "RDTSC instruction not available - timing accuracy compromised"
fi

# Check kernel configuration for real-time
if [ -f /sys/kernel/realtime ]; then
    success "Real-time kernel detected"
elif grep -q "PREEMPT_RT" /boot/config-$(uname -r) 2>/dev/null; then
    success "RT-patched kernel detected"
else
    high "Standard kernel - consider RT kernel for optimal HFT performance"
fi

# Check memory lock limits
ulimit_memlock=$(ulimit -l)
if [ "$ulimit_memlock" = "unlimited" ]; then
    success "Memory lock limit is unlimited"
else
    critical "Memory lock limit too low: $ulimit_memlock (need unlimited for HFT)"
fi

# Check CPU frequency scaling
if [ -f /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor ]; then
    governor=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor)
    if [ "$governor" = "performance" ]; then
        success "CPU frequency scaling set to performance mode"
    else
        high "CPU frequency scaling not optimized: $governor (recommend 'performance')"
    fi
fi

# =============================================================================
# DEPENDENCY VALIDATION
# =============================================================================

log "Validating dependencies..."

# Database connectivity
# PostgreSQL
if command -v psql &> /dev/null; then
    if PGPASSWORD="${DB_PASSWORD:-foxhunt}" psql -h "${DB_HOST:-localhost}" -U "${DB_USER:-foxhunt}" -d "${DB_NAME:-foxhunt}" -c "SELECT version();" &>/dev/null; then
        success "PostgreSQL connection successful"

        # Check database version
        pg_version=$(PGPASSWORD="${DB_PASSWORD:-foxhunt}" psql -h "${DB_HOST:-localhost}" -U "${DB_USER:-foxhunt}" -d "${DB_NAME:-foxhunt}" -t -c "SELECT version();" | head -1)
        log "PostgreSQL version: $pg_version"

        # Check for required extensions
        if PGPASSWORD="${DB_PASSWORD:-foxhunt}" psql -h "${DB_HOST:-localhost}" -U "${DB_USER:-foxhunt}" -d "${DB_NAME:-foxhunt}" -c "SELECT * FROM pg_extension WHERE extname='uuid-ossp';" | grep -q "uuid-ossp"; then
            success "uuid-ossp extension available"
        else
            high "uuid-ossp extension not installed - may cause issues"
        fi
    else
        critical "PostgreSQL connection failed"
    fi
else
    critical "psql command not available"
fi

# Redis
if command -v redis-cli &> /dev/null; then
    if redis-cli -h "${REDIS_HOST:-localhost}" -p "${REDIS_PORT:-6379}" ping | grep -q "PONG"; then
        success "Redis connection successful"

        # Check Redis memory policy
        memory_policy=$(redis-cli -h "${REDIS_HOST:-localhost}" -p "${REDIS_PORT:-6379}" config get maxmemory-policy | tail -1)
        if [ "$memory_policy" = "allkeys-lru" ] || [ "$memory_policy" = "volatile-lru" ]; then
            success "Redis memory policy configured: $memory_policy"
        else
            warning "Redis memory policy not optimized: $memory_policy"
        fi
    else
        critical "Redis connection failed"
    fi
else
    critical "redis-cli command not available"
fi

# InfluxDB
if command -v influx &> /dev/null; then
    if influx ping &>/dev/null; then
        success "InfluxDB connection successful"
    else
        high "InfluxDB connection failed - metrics collection may be impacted"
    fi
else
    warning "InfluxDB CLI not available - cannot validate metrics database"
fi

# =============================================================================
# CONFIGURATION VALIDATION
# =============================================================================

log "Validating configuration files..."

# Check configuration directory exists
if [ -d "$CONFIG_DIR" ]; then
    success "Configuration directory found: $CONFIG_DIR"
else
    critical "Configuration directory not found: $CONFIG_DIR"
fi

# Validate production configuration
PROD_CONFIG="$CONFIG_DIR/production.toml"
if [ -f "$PROD_CONFIG" ]; then
    success "Production configuration file found"

    # Check for required configuration sections
    if grep -q "\[database\]" "$PROD_CONFIG"; then
        success "Database configuration section present"
    else
        critical "Database configuration section missing"
    fi

    if grep -q "\[trading\]" "$PROD_CONFIG"; then
        success "Trading configuration section present"
    else
        critical "Trading configuration section missing"
    fi

    if grep -q "\[risk_management\]" "$PROD_CONFIG"; then
        success "Risk management configuration section present"
    else
        critical "Risk management configuration section missing"
    fi

    # Check for default/insecure values
    if grep -q "password.*=.*\"changeme\"\|secret.*=.*\"default\"\|key.*=.*\"example\"" "$PROD_CONFIG"; then
        critical "Default/insecure credentials found in production configuration"
    else
        success "No default credentials found in production configuration"
    fi
else
    critical "Production configuration file not found: $PROD_CONFIG"
fi

# Check SSL certificates
SSL_CERT_DIR="$CONFIG_DIR/ssl"
if [ -d "$SSL_CERT_DIR" ]; then
    cert_count=$(find "$SSL_CERT_DIR" -name "*.crt" -o -name "*.pem" | wc -l)
    if [ "$cert_count" -gt 0 ]; then
        success "SSL certificates found: $cert_count files"

        # Check certificate expiration
        for cert in "$SSL_CERT_DIR"/*.{crt,pem}; do
            if [ -f "$cert" ]; then
                if openssl x509 -checkend 2592000 -noout -in "$cert" &>/dev/null; then
                    success "Certificate valid for next 30 days: $(basename "$cert")"
                else
                    critical "Certificate expires within 30 days: $(basename "$cert")"
                fi
            fi
        done
    else
        high "No SSL certificates found - HTTPS endpoints may not work"
    fi
else
    warning "SSL certificate directory not found: $SSL_CERT_DIR"
fi

# =============================================================================
# BINARY AND SERVICE VALIDATION
# =============================================================================

log "Validating binaries and services..."

# Check for Foxhunt binaries
FOXHUNT_BIN_DIR="/opt/foxhunt/bin"
if [ -d "$FOXHUNT_BIN_DIR" ]; then
    success "Binary directory found: $FOXHUNT_BIN_DIR"

    REQUIRED_BINARIES=("foxhunt-core" "foxhunt-tli" "foxhunt-risk" "foxhunt-ml" "foxhunt-data")
    for binary in "${REQUIRED_BINARIES[@]}"; do
        if [ -x "$FOXHUNT_BIN_DIR/$binary" ]; then
            success "Binary found and executable: $binary"

            # Check binary version
            if "$FOXHUNT_BIN_DIR/$binary" --version &>/dev/null; then
                version=$("$FOXHUNT_BIN_DIR/$binary" --version 2>/dev/null | head -1)
                log "Version: $binary $version"
            fi
        else
            critical "Binary missing or not executable: $binary"
        fi
    done
else
    critical "Binary directory not found: $FOXHUNT_BIN_DIR"
fi

# Check SystemD service files
SYSTEMD_DIR="/etc/systemd/system"
FOXHUNT_SERVICES=("foxhunt-core" "foxhunt-tli" "foxhunt-risk" "foxhunt-ml" "foxhunt-data")

for service in "${FOXHUNT_SERVICES[@]}"; do
    service_file="$SYSTEMD_DIR/${service}.service"
    if [ -f "$service_file" ]; then
        success "SystemD service file found: $service"

        # Check if service is enabled
        if systemctl is-enabled "$service" &>/dev/null; then
            success "Service enabled: $service"
        else
            warning "Service not enabled: $service"
        fi

        # Validate service file syntax
        if systemd-analyze verify "$service_file" &>/dev/null; then
            success "Service file syntax valid: $service"
        else
            high "Service file syntax issues: $service"
        fi
    else
        critical "SystemD service file missing: $service"
    fi
done

# =============================================================================
# NETWORK AND PORT VALIDATION
# =============================================================================

log "Validating network configuration..."

# Check required ports are available
REQUIRED_PORTS=(8080 8081 8082 8083 8084 50051)
for port in "${REQUIRED_PORTS[@]}"; do
    if netstat -ln | grep -q ":$port "; then
        warning "Port $port is already in use - may conflict with deployment"
    else
        success "Port $port is available"
    fi
done

# Check network performance
if command -v ping &> /dev/null; then
    # Test localhost latency
    localhost_latency=$(ping -c 3 localhost | tail -1 | awk -F '/' '{print $5}')
    if (( $(echo "$localhost_latency < 1.0" | bc -l) )); then
        success "Localhost latency acceptable: ${localhost_latency}ms"
    else
        warning "High localhost latency: ${localhost_latency}ms"
    fi
fi

# =============================================================================
# PERFORMANCE BASELINE VALIDATION
# =============================================================================

log "Establishing performance baseline..."

# Memory bandwidth test (simple)
if command -v dd &> /dev/null; then
    log "Testing memory bandwidth..."
    memory_bandwidth=$(dd if=/dev/zero of=/dev/null bs=1M count=1000 2>&1 | grep -o '[0-9.]* GB/s' | head -1 || echo "unknown")
    log "Memory bandwidth: $memory_bandwidth"
fi

# Disk I/O test
if [ -w "/tmp" ]; then
    log "Testing disk I/O performance..."
    disk_write_speed=$(dd if=/dev/zero of=/tmp/foxhunt_disk_test bs=1M count=100 oflag=direct 2>&1 | grep -o '[0-9.]* MB/s' | tail -1 || echo "unknown")
    rm -f /tmp/foxhunt_disk_test
    log "Disk write speed: $disk_write_speed"
fi

# =============================================================================
# SECURITY VALIDATION
# =============================================================================

log "Validating security configuration..."

# Check file permissions
if [ -f "$PROD_CONFIG" ]; then
    config_perms=$(stat -c "%a" "$PROD_CONFIG" 2>/dev/null || stat -f "%A" "$PROD_CONFIG" 2>/dev/null)
    if [ "$config_perms" = "600" ] || [ "$config_perms" = "0600" ]; then
        success "Production configuration has secure permissions"
    else
        high "Production configuration has insecure permissions: $config_perms"
    fi
fi

# Check for running on privileged ports without proper capabilities
if [ "$(id -u)" -eq 0 ]; then
    warning "Running validation as root - production should use non-root user"
else
    success "Running validation as non-root user"
fi

# Check firewall status
if command -v ufw &> /dev/null; then
    if ufw status | grep -q "Status: active"; then
        success "UFW firewall is active"
    else
        warning "UFW firewall is not active"
    fi
elif command -v firewall-cmd &> /dev/null; then
    if firewall-cmd --state | grep -q "running"; then
        success "Firewalld is running"
    else
        warning "Firewalld is not running"
    fi
fi

# =============================================================================
# DEPLOYMENT SIMULATION
# =============================================================================

log "Running deployment simulation..."

# Check available disk space for releases
if [ -d "/opt/foxhunt/releases" ]; then
    available_space=$(df -h /opt/foxhunt | tail -1 | awk '{print $4}')
    success "Available space for releases: $available_space"
else
    warning "Releases directory not found - will be created during deployment"
fi

# Validate backup capability
if [ -d "/opt/foxhunt/backups" ]; then
    backup_space=$(df -h /opt/foxhunt/backups | tail -1 | awk '{print $4}')
    success "Available backup space: $backup_space"
else
    warning "Backup directory not found - will be created during deployment"
fi

# =============================================================================
# FINAL SUMMARY
# =============================================================================

echo
echo "============================================================"
echo "           PRE-DEPLOYMENT VALIDATION SUMMARY"
echo "============================================================"
echo

log "Validation completed. Generating summary..."

echo -e "Critical Issues: ${RED}$CRITICAL_ISSUES${NC}"
echo -e "High Issues:     ${YELLOW}$HIGH_ISSUES${NC}"
echo -e "Medium Issues:   ${YELLOW}$MEDIUM_ISSUES${NC}"
echo -e "Warnings:        ${YELLOW}$WARNINGS${NC}"
echo

if [ $CRITICAL_ISSUES -eq 0 ] && [ $HIGH_ISSUES -eq 0 ]; then
    echo -e "${GREEN}✅ VALIDATION PASSED${NC}"
    echo
    echo "🚀 System is ready for production deployment!"
    echo
    echo "All critical validations passed. You may proceed with deployment."
    echo "Validation log: $VALIDATION_LOG"
    exit 0
elif [ $CRITICAL_ISSUES -eq 0 ] && [ $HIGH_ISSUES -lt 3 ]; then
    echo -e "${YELLOW}⚠️ VALIDATION PASSED WITH WARNINGS${NC}"
    echo
    echo "🔧 Some high-priority issues detected but deployment can proceed."
    echo
    echo "Review the issues above and consider addressing them."
    echo "Validation log: $VALIDATION_LOG"
    exit 1
else
    echo -e "${RED}❌ VALIDATION FAILED${NC}"
    echo
    echo "🚨 Critical issues must be resolved before deployment!"
    echo
    echo "Address all critical and high-priority issues before proceeding."
    echo "Validation log: $VALIDATION_LOG"
    exit 2
fi