#!/bin/bash # Automated rollback script for Foxhunt HFT Trading System # Version: 3.0.0 # Wave 75 Agent 9 set -euo pipefail BACKUP_DIR="${1:-}" if [ -z "$BACKUP_DIR" ]; then echo "Usage: $0 " echo "Example: $0 /opt/foxhunt/backups/20251003_120000" echo "" echo "Available backups:" ls -1dt /opt/foxhunt/backups/* 2>/dev/null | head -5 || echo " No backups found in /opt/foxhunt/backups" exit 1 fi if [ ! -d "$BACKUP_DIR" ]; then echo "Error: Backup directory $BACKUP_DIR does not exist" exit 1 fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo " FOXHUNT HFT ROLLBACK PROCEDURE" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo " $(date '+%Y-%m-%d %H:%M:%S')" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" echo "Rollback source: $BACKUP_DIR" echo "" # Display backup manifest if available if [ -f "$BACKUP_DIR/manifest.txt" ]; then echo "Backup details:" cat "$BACKUP_DIR/manifest.txt" echo "" fi read -p "This will STOP all services and restore from backup. Continue? (yes/no): " confirm if [ "$confirm" != "yes" ]; then echo "Rollback cancelled" exit 0 fi # Step 1: Stop all services echo "" echo "[1/5] Stopping all services..." sudo systemctl stop foxhunt-trading 2>/dev/null || true sudo systemctl stop foxhunt-backtesting 2>/dev/null || true sudo systemctl stop foxhunt-ml-training 2>/dev/null || true echo "✅ Services stopped" # Step 2: Restore binaries echo "" echo "[2/5] Restoring binaries from backup..." if [ -d "$BACKUP_DIR/bin" ]; then sudo cp -r "$BACKUP_DIR/bin/"* /opt/foxhunt/bin/ sudo chmod 755 /opt/foxhunt/bin/* sudo chown foxhunt:foxhunt /opt/foxhunt/bin/* 2>/dev/null || true echo "✅ Binaries restored" else echo "⚠️ No binaries found in backup, skipping" fi # Step 3: Restore configuration echo "" echo "[3/5] Restoring configuration from backup..." if [ -d "$BACKUP_DIR/config" ]; then sudo cp -r "$BACKUP_DIR/config/"* /opt/foxhunt/config/ 2>/dev/null || true sudo chown -R foxhunt:foxhunt /opt/foxhunt/config 2>/dev/null || true echo "✅ Configuration restored" else echo "⚠️ No configuration found in backup, skipping" fi # Step 4: Restore database (optional, requires manual confirmation) echo "" echo "[4/5] Database rollback..." read -p "Restore database from backup? (yes/no): " restore_db if [ "$restore_db" = "yes" ]; then if [ -f "$BACKUP_DIR/database.sql.gz" ]; then echo "Restoring database (this may take several minutes)..." # Get database connection info DB_NAME="${POSTGRES_DB:-foxhunt_production}" DB_USER="${POSTGRES_USER:-foxhunt_user}" # Drop and recreate database sudo -u postgres psql </dev/null || echo "Warning: Database drop failed (may not exist)" DROP DATABASE IF EXISTS $DB_NAME; CREATE DATABASE $DB_NAME OWNER $DB_USER; EOF # Restore from backup gunzip -c "$BACKUP_DIR/database.sql.gz" | sudo -u postgres psql $DB_NAME echo "✅ Database restored" else echo "❌ Database backup file not found: $BACKUP_DIR/database.sql.gz" echo "Continuing without database restore..." fi else echo "⚠️ Database rollback skipped" fi # Step 5: Restart services echo "" echo "[5/5] Restarting services..." sudo systemctl start foxhunt-trading sleep 5 # Wait for trading service to initialize sudo systemctl start foxhunt-backtesting 2>/dev/null || echo "⚠️ Backtesting service failed to start" sudo systemctl start foxhunt-ml-training 2>/dev/null || echo "⚠️ ML training service failed to start" echo "✅ Services restarted" # Verify health echo "" echo "Running health checks (waiting 10 seconds for initialization)..." sleep 10 if [ -x "/opt/foxhunt/bin/health_check.sh" ]; then /opt/foxhunt/bin/health_check.sh elif [ -x "$(dirname $0)/health_check.sh" ]; then $(dirname $0)/health_check.sh else echo "⚠️ Health check script not found, manual verification required" fi echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo " ✅ ROLLBACK COMPLETE" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" echo "Next steps:" echo " 1. Verify all services are healthy" echo " 2. Check logs for errors: journalctl -u foxhunt-* --since '5 minutes ago'" echo " 3. Monitor performance for 30 minutes" echo " 4. Document incident and rollback in postmortem"