#!/bin/bash # FOXHUNT HFT SYSTEM - PRODUCTION ROLLBACK SCRIPT # Emergency rollback for financial trading system # CRITICAL: Use only for emergency situations with real money impact set -euo pipefail # Colors and formatting RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' BOLD='\033[1m' NC='\033[0m' # Global variables SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" ROLLBACK_LOG="/var/log/foxhunt/emergency-rollback-$(date +%Y%m%d-%H%M%S).log" BACKUP_DIR="/opt/foxhunt/backups" CURRENT_VERSION="" TARGET_VERSION="" EMERGENCY_MODE=false # Create log directory mkdir -p "$(dirname "$ROLLBACK_LOG")" exec > >(tee -a "$ROLLBACK_LOG") exec 2>&1 # Logging functions log_critical() { echo -e "${RED}${BOLD}[CRITICAL]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" } log_info() { echo -e "${BLUE}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" } log_success() { echo -e "${GREEN}[SUCCESS]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" } log_warning() { echo -e "${YELLOW}[WARNING]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" } # Emergency banner print_emergency_banner() { cat << 'EOF' ╔══════════════════════════════════════════════════════════════╗ ║ 🚨 EMERGENCY ROLLBACK 🚨 ║ ║ ║ ║ FOXHUNT HFT SYSTEM - CRITICAL ACTION ║ ║ ║ ║ ⚠️ REAL MONEY TRADING SYSTEM ROLLBACK IN PROGRESS ⚠️ ║ ║ ║ ║ This script will: ║ ║ • Stop all trading operations immediately ║ ║ • Rollback to previous stable version ║ ║ • Restore database to last known good state ║ ║ • Notify all stakeholders ║ ║ ║ ║ Use only in genuine emergency situations! ║ ║ ║ ╚══════════════════════════════════════════════════════════════╝ EOF } # Usage information show_usage() { cat << EOF Usage: $0 [OPTIONS] Emergency rollback script for Foxhunt HFT production system. OPTIONS: -v, --version VERSION Target version to rollback to -e, --emergency Enable emergency mode (skip confirmations) -d, --dry-run Simulate rollback without making changes -h, --help Show this help message EXAMPLES: $0 --version v0.9.5 # Rollback to specific version $0 --emergency --version v0.9.5 # Emergency rollback (no prompts) $0 --dry-run --version v0.9.5 # Test rollback procedure EMERGENCY HOTLINE: +1-555-TRADING (24/7) EOF } # Parse command line arguments parse_arguments() { while [[ $# -gt 0 ]]; do case $1 in -v|--version) TARGET_VERSION="$2" shift 2 ;; -e|--emergency) EMERGENCY_MODE=true shift ;; -d|--dry-run) DRY_RUN=true shift ;; -h|--help) show_usage exit 0 ;; *) log_critical "Unknown option: $1" show_usage exit 1 ;; esac done } # Emergency confirmation emergency_confirmation() { if [[ "$EMERGENCY_MODE" == "true" ]]; then log_critical "EMERGENCY MODE: Skipping confirmations" return 0 fi echo "" echo -e "${RED}${BOLD}⚠️ CRITICAL CONFIRMATION REQUIRED ⚠️${NC}" echo "" echo "This action will:" echo "1. 🛑 STOP ALL LIVE TRADING OPERATIONS" echo "2. 🔄 ROLLBACK TO VERSION: $TARGET_VERSION" echo "3. 🗃️ RESTORE DATABASE FROM BACKUP" echo "4. 📧 NOTIFY ALL STAKEHOLDERS" echo "5. 📊 IMPACT LIVE TRADING POSITIONS" echo "" echo "Current system version: $CURRENT_VERSION" echo "Target rollback version: $TARGET_VERSION" echo "" read -p "Do you understand the impact? (type 'YES I UNDERSTAND'): " confirmation if [[ "$confirmation" != "YES I UNDERSTAND" ]]; then log_critical "Rollback cancelled by user" exit 1 fi read -p "Enter incident ticket number: " incident_number if [[ -z "$incident_number" ]]; then log_critical "Incident number required for audit trail" exit 1 fi echo "INCIDENT_NUMBER=$incident_number" >> "$ROLLBACK_LOG" } # Stop trading operations stop_trading_operations() { log_critical "STOPPING ALL TRADING OPERATIONS" # Set maintenance mode curl -X POST http://localhost:8081/maintenance/enable \ -H "Content-Type: application/json" \ -d '{"reason": "Emergency rollback", "estimated_duration": "30m"}' \ || log_warning "Failed to set maintenance mode via API" # Stop trading engine gracefully log_info "Gracefully stopping trading engine..." docker-compose -f "$PROJECT_ROOT/docker-compose.final-production.yaml" exec trading-engine killall -TERM trading-engine || true sleep 10 # Force stop if still running docker-compose -f "$PROJECT_ROOT/docker-compose.final-production.yaml" stop trading-engine # Stop other critical services local services=("broker-connector" "risk-management" "market-data") for service in "${services[@]}"; do log_info "Stopping $service..." docker-compose -f "$PROJECT_ROOT/docker-compose.final-production.yaml" stop "$service" done log_success "All trading operations stopped" } # Create emergency backup create_emergency_backup() { log_info "Creating emergency backup before rollback..." local backup_timestamp=$(date +%Y%m%d_%H%M%S) local emergency_backup_dir="$BACKUP_DIR/emergency_rollback_$backup_timestamp" mkdir -p "$emergency_backup_dir" # Backup database log_info "Backing up current database..." docker-compose -f "$PROJECT_ROOT/docker-compose.final-production.yaml" exec -T postgres-primary \ pg_dump -U foxhunt_trading_user -d foxhunt_trading --verbose \ > "$emergency_backup_dir/database_pre_rollback.sql" # Backup Redis data log_info "Backing up Redis data..." docker-compose -f "$PROJECT_ROOT/docker-compose.final-production.yaml" exec -T redis-master \ redis-cli --rdb "$emergency_backup_dir/redis_pre_rollback.rdb" # Backup configuration files log_info "Backing up configuration files..." cp -r "$PROJECT_ROOT/.env.production" "$emergency_backup_dir/" cp -r "$PROJECT_ROOT/certs" "$emergency_backup_dir/" # Create backup manifest cat > "$emergency_backup_dir/manifest.txt" << EOF Emergency Backup Manifest ======================== Timestamp: $(date '+%Y-%m-%d %H:%M:%S %Z') Current Version: $CURRENT_VERSION Target Version: $TARGET_VERSION Operator: $(whoami) Hostname: $(hostname) Reason: Emergency rollback Files: - database_pre_rollback.sql: Complete database dump - redis_pre_rollback.rdb: Redis data snapshot - .env.production: Environment configuration - certs/: TLS certificates This backup was created automatically before emergency rollback. EOF log_success "Emergency backup created: $emergency_backup_dir" echo "EMERGENCY_BACKUP_PATH=$emergency_backup_dir" >> "$ROLLBACK_LOG" } # Rollback database rollback_database() { log_info "Rolling back database to version $TARGET_VERSION..." local target_backup="$BACKUP_DIR/database_$TARGET_VERSION.sql" if [[ ! -f "$target_backup" ]]; then log_critical "Database backup for version $TARGET_VERSION not found: $target_backup" return 1 fi # Stop database connections docker-compose -f "$PROJECT_ROOT/docker-compose.final-production.yaml" exec -T postgres-primary \ psql -U foxhunt_trading_user -d postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'foxhunt_trading';" # Drop and recreate database docker-compose -f "$PROJECT_ROOT/docker-compose.final-production.yaml" exec -T postgres-primary \ psql -U foxhunt_trading_user -d postgres -c "DROP DATABASE IF EXISTS foxhunt_trading;" docker-compose -f "$PROJECT_ROOT/docker-compose.final-production.yaml" exec -T postgres-primary \ psql -U foxhunt_trading_user -d postgres -c "CREATE DATABASE foxhunt_trading;" # Restore from backup docker-compose -f "$PROJECT_ROOT/docker-compose.final-production.yaml" exec -T postgres-primary \ psql -U foxhunt_trading_user -d foxhunt_trading < "$target_backup" log_success "Database rolled back successfully" } # Rollback application services rollback_services() { log_info "Rolling back services to version $TARGET_VERSION..." # Update environment to use target version sed -i "s/VERSION=.*/VERSION=$TARGET_VERSION/" "$PROJECT_ROOT/.env.production" # Pull target images docker-compose -f "$PROJECT_ROOT/docker-compose.final-production.yaml" pull # Start services in correct order log_info "Starting infrastructure services..." docker-compose -f "$PROJECT_ROOT/docker-compose.final-production.yaml" up -d postgres-primary redis-master sleep 10 log_info "Starting monitoring services..." docker-compose -f "$PROJECT_ROOT/docker-compose.final-production.yaml" up -d prometheus grafana sleep 5 log_info "Starting trading services..." docker-compose -f "$PROJECT_ROOT/docker-compose.final-production.yaml" up -d market-data broker-connector risk-management sleep 10 log_info "Starting trading engine..." docker-compose -f "$PROJECT_ROOT/docker-compose.final-production.yaml" up -d trading-engine sleep 15 log_info "Starting load balancer..." docker-compose -f "$PROJECT_ROOT/docker-compose.final-production.yaml" up -d nginx log_success "Services rolled back to version $TARGET_VERSION" } # Verify rollback success verify_rollback() { log_info "Verifying rollback success..." local failed_checks=() # Check service health local services=("trading-engine:8081" "broker-connector:8080" "market-data:8084" "risk-management:8085") for service_check in "${services[@]}"; do IFS=':' read -r service port <<< "$service_check" local max_attempts=30 local attempt=1 while [[ $attempt -le $max_attempts ]]; do if curl -f -s "http://localhost:$port/health" > /dev/null; then log_success "$service health check passed" break fi if [[ $attempt -eq $max_attempts ]]; then failed_checks+=("$service") break fi sleep 2 ((attempt++)) done done # Check database connectivity if ! docker-compose -f "$PROJECT_ROOT/docker-compose.final-production.yaml" exec -T postgres-primary \ psql -U foxhunt_trading_user -d foxhunt_trading -c "SELECT 1;" > /dev/null; then failed_checks+=("database") fi # Check version local actual_version actual_version=$(curl -s http://localhost:8081/version | jq -r '.version' 2>/dev/null || echo "unknown") if [[ "$actual_version" != "$TARGET_VERSION" ]]; then failed_checks+=("version_mismatch:$actual_version") fi if [[ ${#failed_checks[@]} -gt 0 ]]; then log_critical "Rollback verification failed: ${failed_checks[*]}" return 1 fi log_success "Rollback verification completed successfully" } # Notify stakeholders notify_stakeholders() { log_info "Notifying stakeholders of rollback completion..." local notification_message=" 🚨 FOXHUNT HFT SYSTEM - EMERGENCY ROLLBACK COMPLETED 🚨 System Status: OPERATIONAL (Rolled Back) Previous Version: $CURRENT_VERSION Current Version: $TARGET_VERSION Rollback Time: $(date '+%Y-%m-%d %H:%M:%S %Z') Operator: $(whoami) Duration: $SECONDS seconds Trading operations have been restored and are operational. Please monitor system performance closely. Emergency Backup: $EMERGENCY_BACKUP_PATH Rollback Log: $ROLLBACK_LOG Next Steps: 1. Monitor system performance 2. Validate trading operations 3. Update incident documentation 4. Schedule post-incident review Emergency Hotline: +1-555-TRADING " # Send email notifications (if configured) if command -v mail &> /dev/null && [[ -n "${ALERT_EMAIL_TO:-}" ]]; then echo "$notification_message" | mail -s "🚨 Foxhunt HFT - Emergency Rollback Completed" "$ALERT_EMAIL_TO" fi # Slack notification (if configured) if [[ -n "${SLACK_WEBHOOK_URL:-}" ]]; then curl -X POST -H 'Content-type: application/json' \ --data "{\"text\":\"$notification_message\"}" \ "$SLACK_WEBHOOK_URL" 2>/dev/null || log_warning "Failed to send Slack notification" fi # Log notification log_success "Stakeholder notifications sent" } # Generate rollback report generate_rollback_report() { log_info "Generating rollback report..." local report_file="/var/log/foxhunt/rollback-report-$(date +%Y%m%d-%H%M%S).md" cat > "$report_file" << EOF # Foxhunt HFT System - Emergency Rollback Report ## Incident Summary **Date:** $(date '+%Y-%m-%d %H:%M:%S %Z') **Type:** Emergency System Rollback **Operator:** $(whoami) **Hostname:** $(hostname) **Duration:** $SECONDS seconds ## Version Information - **Previous Version:** $CURRENT_VERSION - **Rolled Back To:** $TARGET_VERSION - **Rollback Reason:** Emergency incident ## Actions Performed 1. ✅ Stopped all trading operations 2. ✅ Created emergency backup 3. ✅ Rolled back database 4. ✅ Rolled back application services 5. ✅ Verified system health 6. ✅ Notified stakeholders ## System Status $(docker-compose -f "$PROJECT_ROOT/docker-compose.final-production.yaml" ps) ## Emergency Backup **Location:** $EMERGENCY_BACKUP_PATH **Contents:** - Complete database dump - Redis data snapshot - Configuration files - TLS certificates ## Post-Rollback Verification - ✅ All services healthy - ✅ Database connectivity verified - ✅ Version confirmation: $TARGET_VERSION - ✅ Trading operations restored ## Impact Assessment - **Trading Downtime:** $SECONDS seconds - **Data Loss:** None (backup created) - **Service Availability:** Fully restored - **Financial Impact:** To be determined ## Next Steps 1. **Immediate (0-1 hours):** - Monitor system performance - Validate all trading operations - Check order book consistency - Verify position accuracy 2. **Short-term (1-24 hours):** - Conduct thorough testing - Update incident documentation - Communicate with clients (if necessary) - Prepare detailed impact analysis 3. **Medium-term (1-7 days):** - Schedule post-incident review - Analyze root cause - Update rollback procedures - Implement preventive measures ## Contact Information - **Emergency Hotline:** +1-555-TRADING - **Technical Support:** trading-ops@foxhunt.com - **Incident Manager:** TBD --- **Report generated by:** Foxhunt Emergency Rollback Script v1.0.0 **Log File:** $ROLLBACK_LOG **Emergency Backup:** $EMERGENCY_BACKUP_PATH EOF log_success "Rollback report generated: $report_file" } # Main rollback execution main() { # Get current version CURRENT_VERSION=$(docker-compose -f "$PROJECT_ROOT/docker-compose.final-production.yaml" exec -T trading-engine /app/trading-engine --version 2>/dev/null | head -1 || echo "unknown") print_emergency_banner log_critical "EMERGENCY ROLLBACK INITIATED" log_info "Current version: $CURRENT_VERSION" log_info "Target version: $TARGET_VERSION" log_info "Operator: $(whoami)" log_info "Timestamp: $(date '+%Y-%m-%d %H:%M:%S %Z')" # Confirm emergency action emergency_confirmation # Execute rollback phases stop_trading_operations create_emergency_backup rollback_database rollback_services verify_rollback notify_stakeholders generate_rollback_report log_success "🎉 EMERGENCY ROLLBACK COMPLETED SUCCESSFULLY! 🎉" log_info "System rolled back from $CURRENT_VERSION to $TARGET_VERSION" log_info "Total rollback time: $SECONDS seconds" log_info "All services are operational and healthy" echo "" echo -e "${GREEN}${BOLD}✅ ROLLBACK SUCCESSFUL${NC}" echo "" echo "System Status: OPERATIONAL" echo "Current Version: $TARGET_VERSION" echo "Rollback Duration: $SECONDS seconds" echo "" echo "🔗 Service URLs:" echo " • Trading Dashboard: http://localhost:3000" echo " • Trading Engine API: http://localhost:8081" echo "" echo "📋 Important Files:" echo " • Rollback Log: $ROLLBACK_LOG" echo " • Emergency Backup: $EMERGENCY_BACKUP_PATH" echo "" echo -e "${YELLOW}⚠️ POST-ROLLBACK ACTIONS REQUIRED:${NC}" echo " 1. Monitor system performance continuously" echo " 2. Validate all trading operations" echo " 3. Check position consistency" echo " 4. Update incident documentation" echo " 5. Schedule post-incident review" echo "" echo "📞 Emergency Support: +1-555-TRADING (24/7)" } # Ensure target version is specified if [[ -z "${TARGET_VERSION:-}" ]]; then log_critical "Target version must be specified with --version" show_usage exit 1 fi # Signal handlers trap 'log_critical "Rollback interrupted! System may be in inconsistent state!"; exit 130' INT TERM # Change to project root cd "$PROJECT_ROOT" || { log_critical "Failed to change to project root directory"; exit 1; } # Parse arguments and execute parse_arguments "$@" main