#!/bin/bash # Pre-Deployment Validation Script for Foxhunt HFT Trading System # Comprehensive pre-flight checks for zero-downtime deployment # # This script validates all dependencies, configurations, and system readiness # before initiating a production deployment. set -euo pipefail # Configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" FOXHUNT_HOME="/opt/foxhunt" CONFIG_DIR="${FOXHUNT_HOME}/config" VALIDATION_LOG="/home/jgrusewski/Work/foxhunt/logs/pre-deployment-$(date +%s).log" # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # Validation counters CRITICAL_ISSUES=0 HIGH_ISSUES=0 MEDIUM_ISSUES=0 WARNINGS=0 # Create log directory if it doesn't exist mkdir -p "$(dirname "$VALIDATION_LOG")" log() { echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1" | tee -a "$VALIDATION_LOG" } error() { echo -e "${RED}[ERROR]${NC} $1" | tee -a "$VALIDATION_LOG" ((CRITICAL_ISSUES++)) } warning() { echo -e "${YELLOW}[WARNING]${NC} $1" | tee -a "$VALIDATION_LOG" ((WARNINGS++)) } success() { echo -e "${GREEN}[SUCCESS]${NC} $1" | tee -a "$VALIDATION_LOG" } critical() { echo -e "${RED}[CRITICAL]${NC} $1" | tee -a "$VALIDATION_LOG" ((CRITICAL_ISSUES++)) } high() { echo -e "${YELLOW}[HIGH]${NC} $1" | tee -a "$VALIDATION_LOG" ((HIGH_ISSUES++)) } # Print banner echo "============================================================" echo " Foxhunt HFT Pre-Deployment Validation" echo "============================================================" echo # ============================================================================= # SYSTEM REQUIREMENTS VALIDATION # ============================================================================= log "Validating system requirements..." # Check CPU architecture and features if grep -q "avx2" /proc/cpuinfo; then success "AVX2 instruction set available for SIMD optimizations" else critical "AVX2 instruction set not available - HFT performance will be degraded" fi if grep -q "rdtsc" /proc/cpuinfo; then success "RDTSC instruction available for nanosecond timing" else critical "RDTSC instruction not available - timing accuracy compromised" fi # Check kernel configuration for real-time if [ -f /sys/kernel/realtime ]; then success "Real-time kernel detected" elif grep -q "PREEMPT_RT" /boot/config-$(uname -r) 2>/dev/null; then success "RT-patched kernel detected" else high "Standard kernel - consider RT kernel for optimal HFT performance" fi # Check memory lock limits ulimit_memlock=$(ulimit -l) if [ "$ulimit_memlock" = "unlimited" ]; then success "Memory lock limit is unlimited" else critical "Memory lock limit too low: $ulimit_memlock (need unlimited for HFT)" fi # Check CPU frequency scaling if [ -f /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor ]; then governor=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor) if [ "$governor" = "performance" ]; then success "CPU frequency scaling set to performance mode" else high "CPU frequency scaling not optimized: $governor (recommend 'performance')" fi fi # ============================================================================= # DEPENDENCY VALIDATION # ============================================================================= log "Validating dependencies..." # Database connectivity # PostgreSQL if command -v psql &> /dev/null; then if PGPASSWORD="${DB_PASSWORD:-foxhunt}" psql -h "${DB_HOST:-localhost}" -U "${DB_USER:-foxhunt}" -d "${DB_NAME:-foxhunt}" -c "SELECT version();" &>/dev/null; then success "PostgreSQL connection successful" # Check database version pg_version=$(PGPASSWORD="${DB_PASSWORD:-foxhunt}" psql -h "${DB_HOST:-localhost}" -U "${DB_USER:-foxhunt}" -d "${DB_NAME:-foxhunt}" -t -c "SELECT version();" | head -1) log "PostgreSQL version: $pg_version" # Check for required extensions if PGPASSWORD="${DB_PASSWORD:-foxhunt}" psql -h "${DB_HOST:-localhost}" -U "${DB_USER:-foxhunt}" -d "${DB_NAME:-foxhunt}" -c "SELECT * FROM pg_extension WHERE extname='uuid-ossp';" | grep -q "uuid-ossp"; then success "uuid-ossp extension available" else high "uuid-ossp extension not installed - may cause issues" fi else critical "PostgreSQL connection failed" fi else critical "psql command not available" fi # Redis if command -v redis-cli &> /dev/null; then if redis-cli -h "${REDIS_HOST:-localhost}" -p "${REDIS_PORT:-6379}" ping | grep -q "PONG"; then success "Redis connection successful" # Check Redis memory policy memory_policy=$(redis-cli -h "${REDIS_HOST:-localhost}" -p "${REDIS_PORT:-6379}" config get maxmemory-policy | tail -1) if [ "$memory_policy" = "allkeys-lru" ] || [ "$memory_policy" = "volatile-lru" ]; then success "Redis memory policy configured: $memory_policy" else warning "Redis memory policy not optimized: $memory_policy" fi else critical "Redis connection failed" fi else critical "redis-cli command not available" fi # InfluxDB if command -v influx &> /dev/null; then if influx ping &>/dev/null; then success "InfluxDB connection successful" else high "InfluxDB connection failed - metrics collection may be impacted" fi else warning "InfluxDB CLI not available - cannot validate metrics database" fi # ============================================================================= # CONFIGURATION VALIDATION # ============================================================================= log "Validating configuration files..." # Check configuration directory exists if [ -d "$CONFIG_DIR" ]; then success "Configuration directory found: $CONFIG_DIR" else critical "Configuration directory not found: $CONFIG_DIR" fi # Validate production configuration PROD_CONFIG="$CONFIG_DIR/production.toml" if [ -f "$PROD_CONFIG" ]; then success "Production configuration file found" # Check for required configuration sections if grep -q "\[database\]" "$PROD_CONFIG"; then success "Database configuration section present" else critical "Database configuration section missing" fi if grep -q "\[trading\]" "$PROD_CONFIG"; then success "Trading configuration section present" else critical "Trading configuration section missing" fi if grep -q "\[risk_management\]" "$PROD_CONFIG"; then success "Risk management configuration section present" else critical "Risk management configuration section missing" fi # Check for default/insecure values if grep -q "password.*=.*\"changeme\"\|secret.*=.*\"default\"\|key.*=.*\"example\"" "$PROD_CONFIG"; then critical "Default/insecure credentials found in production configuration" else success "No default credentials found in production configuration" fi else critical "Production configuration file not found: $PROD_CONFIG" fi # Check SSL certificates SSL_CERT_DIR="$CONFIG_DIR/ssl" if [ -d "$SSL_CERT_DIR" ]; then cert_count=$(find "$SSL_CERT_DIR" -name "*.crt" -o -name "*.pem" | wc -l) if [ "$cert_count" -gt 0 ]; then success "SSL certificates found: $cert_count files" # Check certificate expiration for cert in "$SSL_CERT_DIR"/*.{crt,pem}; do if [ -f "$cert" ]; then if openssl x509 -checkend 2592000 -noout -in "$cert" &>/dev/null; then success "Certificate valid for next 30 days: $(basename "$cert")" else critical "Certificate expires within 30 days: $(basename "$cert")" fi fi done else high "No SSL certificates found - HTTPS endpoints may not work" fi else warning "SSL certificate directory not found: $SSL_CERT_DIR" fi # ============================================================================= # BINARY AND SERVICE VALIDATION # ============================================================================= log "Validating binaries and services..." # Check for Foxhunt binaries FOXHUNT_BIN_DIR="/opt/foxhunt/bin" if [ -d "$FOXHUNT_BIN_DIR" ]; then success "Binary directory found: $FOXHUNT_BIN_DIR" REQUIRED_BINARIES=("foxhunt-core" "foxhunt-tli" "foxhunt-risk" "foxhunt-ml" "foxhunt-data") for binary in "${REQUIRED_BINARIES[@]}"; do if [ -x "$FOXHUNT_BIN_DIR/$binary" ]; then success "Binary found and executable: $binary" # Check binary version if "$FOXHUNT_BIN_DIR/$binary" --version &>/dev/null; then version=$("$FOXHUNT_BIN_DIR/$binary" --version 2>/dev/null | head -1) log "Version: $binary $version" fi else critical "Binary missing or not executable: $binary" fi done else critical "Binary directory not found: $FOXHUNT_BIN_DIR" fi # Check SystemD service files SYSTEMD_DIR="/etc/systemd/system" FOXHUNT_SERVICES=("foxhunt-core" "foxhunt-tli" "foxhunt-risk" "foxhunt-ml" "foxhunt-data") for service in "${FOXHUNT_SERVICES[@]}"; do service_file="$SYSTEMD_DIR/${service}.service" if [ -f "$service_file" ]; then success "SystemD service file found: $service" # Check if service is enabled if systemctl is-enabled "$service" &>/dev/null; then success "Service enabled: $service" else warning "Service not enabled: $service" fi # Validate service file syntax if systemd-analyze verify "$service_file" &>/dev/null; then success "Service file syntax valid: $service" else high "Service file syntax issues: $service" fi else critical "SystemD service file missing: $service" fi done # ============================================================================= # NETWORK AND PORT VALIDATION # ============================================================================= log "Validating network configuration..." # Check required ports are available REQUIRED_PORTS=(8080 8081 8082 8083 8084 50051) for port in "${REQUIRED_PORTS[@]}"; do if netstat -ln | grep -q ":$port "; then warning "Port $port is already in use - may conflict with deployment" else success "Port $port is available" fi done # Check network performance if command -v ping &> /dev/null; then # Test localhost latency localhost_latency=$(ping -c 3 localhost | tail -1 | awk -F '/' '{print $5}') if (( $(echo "$localhost_latency < 1.0" | bc -l) )); then success "Localhost latency acceptable: ${localhost_latency}ms" else warning "High localhost latency: ${localhost_latency}ms" fi fi # ============================================================================= # PERFORMANCE BASELINE VALIDATION # ============================================================================= log "Establishing performance baseline..." # Memory bandwidth test (simple) if command -v dd &> /dev/null; then log "Testing memory bandwidth..." memory_bandwidth=$(dd if=/dev/zero of=/dev/null bs=1M count=1000 2>&1 | grep -o '[0-9.]* GB/s' | head -1 || echo "unknown") log "Memory bandwidth: $memory_bandwidth" fi # Disk I/O test if [ -w "/tmp" ]; then log "Testing disk I/O performance..." disk_write_speed=$(dd if=/dev/zero of=/tmp/foxhunt_disk_test bs=1M count=100 oflag=direct 2>&1 | grep -o '[0-9.]* MB/s' | tail -1 || echo "unknown") rm -f /tmp/foxhunt_disk_test log "Disk write speed: $disk_write_speed" fi # ============================================================================= # SECURITY VALIDATION # ============================================================================= log "Validating security configuration..." # Check file permissions if [ -f "$PROD_CONFIG" ]; then config_perms=$(stat -c "%a" "$PROD_CONFIG" 2>/dev/null || stat -f "%A" "$PROD_CONFIG" 2>/dev/null) if [ "$config_perms" = "600" ] || [ "$config_perms" = "0600" ]; then success "Production configuration has secure permissions" else high "Production configuration has insecure permissions: $config_perms" fi fi # Check for running on privileged ports without proper capabilities if [ "$(id -u)" -eq 0 ]; then warning "Running validation as root - production should use non-root user" else success "Running validation as non-root user" fi # Check firewall status if command -v ufw &> /dev/null; then if ufw status | grep -q "Status: active"; then success "UFW firewall is active" else warning "UFW firewall is not active" fi elif command -v firewall-cmd &> /dev/null; then if firewall-cmd --state | grep -q "running"; then success "Firewalld is running" else warning "Firewalld is not running" fi fi # ============================================================================= # DEPLOYMENT SIMULATION # ============================================================================= log "Running deployment simulation..." # Check available disk space for releases if [ -d "/opt/foxhunt/releases" ]; then available_space=$(df -h /opt/foxhunt | tail -1 | awk '{print $4}') success "Available space for releases: $available_space" else warning "Releases directory not found - will be created during deployment" fi # Validate backup capability if [ -d "/opt/foxhunt/backups" ]; then backup_space=$(df -h /opt/foxhunt/backups | tail -1 | awk '{print $4}') success "Available backup space: $backup_space" else warning "Backup directory not found - will be created during deployment" fi # ============================================================================= # FINAL SUMMARY # ============================================================================= echo echo "============================================================" echo " PRE-DEPLOYMENT VALIDATION SUMMARY" echo "============================================================" echo log "Validation completed. Generating summary..." echo -e "Critical Issues: ${RED}$CRITICAL_ISSUES${NC}" echo -e "High Issues: ${YELLOW}$HIGH_ISSUES${NC}" echo -e "Medium Issues: ${YELLOW}$MEDIUM_ISSUES${NC}" echo -e "Warnings: ${YELLOW}$WARNINGS${NC}" echo if [ $CRITICAL_ISSUES -eq 0 ] && [ $HIGH_ISSUES -eq 0 ]; then echo -e "${GREEN}✅ VALIDATION PASSED${NC}" echo echo "🚀 System is ready for production deployment!" echo echo "All critical validations passed. You may proceed with deployment." echo "Validation log: $VALIDATION_LOG" exit 0 elif [ $CRITICAL_ISSUES -eq 0 ] && [ $HIGH_ISSUES -lt 3 ]; then echo -e "${YELLOW}⚠️ VALIDATION PASSED WITH WARNINGS${NC}" echo echo "🔧 Some high-priority issues detected but deployment can proceed." echo echo "Review the issues above and consider addressing them." echo "Validation log: $VALIDATION_LOG" exit 1 else echo -e "${RED}❌ VALIDATION FAILED${NC}" echo echo "🚨 Critical issues must be resolved before deployment!" echo echo "Address all critical and high-priority issues before proceeding." echo "Validation log: $VALIDATION_LOG" exit 2 fi