foxhunt/deployment/scripts/blue-green-deploy.sh

#!/bin/bash
# Blue-green deployment script for Foxhunt HFT Trading System
# Implements zero-downtime deployment with instant traffic switching

set -euo pipefail

# Configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
DEPLOYMENT_LOG="/home/jgrusewski/Work/foxhunt/logs/blue-green-deployment-$(date +%s).log"
FOXHUNT_HOME="/opt/foxhunt"
RELEASES_DIR="/opt/foxhunt/releases"
CURRENT_LINK="/opt/foxhunt/current"
GREEN_LINK="/opt/foxhunt/green"
BLUE_LINK="/opt/foxhunt/blue"
LOAD_BALANCER_CONFIG="/etc/nginx/sites-available/foxhunt-lb"

# Services in dependency order
SERVICES=("foxhunt-core" "foxhunt-data" "foxhunt-risk" "foxhunt-ml" "foxhunt-tli")

# Performance thresholds
MAX_LATENCY_US=30
MIN_THROUGHPUT_OPS=1000

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$DEPLOYMENT_LOG"
}

error() {
    echo -e "${RED}ERROR: $1${NC}" | tee -a "$DEPLOYMENT_LOG"
}

success() {
    echo -e "${GREEN}SUCCESS: $1${NC}" | tee -a "$DEPLOYMENT_LOG"
}

warning() {
    echo -e "${YELLOW}WARNING: $1${NC}" | tee -a "$DEPLOYMENT_LOG"
}

info() {
    echo -e "${BLUE}INFO: $1${NC}" | tee -a "$DEPLOYMENT_LOG"
}

cleanup() {
    log "Cleaning up blue-green deployment artifacts..."
    exit "${1:-1}"
}

trap cleanup EXIT INT TERM

usage() {
    echo "Usage: $0 <version> [options]"
    echo "Options:"
    echo "  --validate-only      Only validate deployment, don't deploy"
    echo "  --skip-performance   Skip performance validation"
    echo "  --keep-old          Keep old environment after successful deployment"
    exit 1
}

validate_performance() {
    local service_name="$1"
    local endpoint="$2"
    
    log "Validating performance for $service_name..."
    
    # Check latency with multiple samples
    local total_latency=0
    local samples=10
    
    for i in $(seq 1 $samples); do
        local start_time=$(date +%s%N)
        if curl -f -s "$endpoint/health" > /dev/null 2>&1; then
            local end_time=$(date +%s%N)
            local latency_ns=$((end_time - start_time))
            local latency_us=$((latency_ns / 1000))
            total_latency=$((total_latency + latency_us))
        else
            error "Health check failed for $service_name"
            return 1
        fi
        sleep 0.1
    done
    
    local avg_latency_us=$((total_latency / samples))
    
    if [ "$avg_latency_us" -gt "$MAX_LATENCY_US" ]; then
        error "Average latency validation failed: ${avg_latency_us}μs > ${MAX_LATENCY_US}μs threshold"
        return 1
    fi
    
    # Check system metrics
    local cpu_usage
    cpu_usage=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | cut -d'%' -f1)
    
    if (( $(echo "$cpu_usage > 80" | bc -l) )); then
        warning "High CPU usage detected: ${cpu_usage}%"
    fi
    
    success "Performance validation passed - Average latency: ${avg_latency_us}μs, CPU: ${cpu_usage}%"
    return 0
}

health_check() {
    local service_name="$1"
    local health_url="$2"
    local max_attempts=30
    local attempt=0
    
    log "Health checking $service_name at $health_url..."
    
    while [ $attempt -lt $max_attempts ]; do
        if curl -f -s "$health_url" > /dev/null 2>&1; then
            # Additional gRPC check for TLI
            if [[ "$service_name" == *"tli"* ]]; then
                local grpc_port=$(echo "$health_url" | sed 's/.*:\([0-9]*\).*/\1/')
                local grpc_check_port=$((grpc_port + 1000))  # Assume gRPC is on different port
                if grpcurl -plaintext "localhost:$grpc_check_port" list > /dev/null 2>&1; then
                    success "$service_name is healthy (HTTP + gRPC)"
                    return 0
                fi
            else
                success "$service_name is healthy"
                return 0
            fi
        fi
        
        attempt=$((attempt + 1))
        log "Health check attempt $attempt/$max_attempts failed, retrying in 3s..."
        sleep 3
    done
    
    error "$service_name failed health check after $max_attempts attempts"
    return 1
}

determine_current_environment() {
    log "Determining current active environment..."
    
    if [ -L "$CURRENT_LINK" ]; then
        local current_target
        current_target=$(readlink "$CURRENT_LINK")
        
        if [[ "$current_target" == *"blue"* ]]; then
            echo "blue"
        elif [[ "$current_target" == *"green"* ]]; then
            echo "green"
        else
            # Default to blue if unclear
            echo "blue"
        fi
    else
        # No current deployment, default to blue
        echo "blue"
    fi
}

setup_environment() {
    local env_name="$1"
    local version="$2"
    local release_dir="$RELEASES_DIR/$version"
    local env_link="$FOXHUNT_HOME/$env_name"
    
    log "Setting up $env_name environment with version $version..."
    
    # Create environment directory
    mkdir -p "$env_link"
    
    # Copy release to environment
    rsync -a --delete "$release_dir/" "$env_link/"
    
    # Update Docker Compose for this environment
    local compose_file="$env_link/docker-compose.${env_name}.yml"
    cp "$FOXHUNT_HOME/docker/docker-compose.production.yml" "$compose_file"
    
    # Update container names to include environment
    sed -i "s/foxhunt-\([^-]*\)-prod/foxhunt-\1-${env_name}/g" "$compose_file"
    
    # Update port mappings to avoid conflicts
    case $env_name in
        blue)
            # Blue uses standard ports (8080, 8081, etc.)
            ;;
        green)
            # Green uses offset ports (8180, 8181, etc.)
            sed -i 's/:808\([0-9]\)/:818\1/g' "$compose_file"
            sed -i 's/:909\([0-9]\)/:919\1/g' "$compose_file"
            sed -i 's/:5005\([0-9]\)/:5105\1/g' "$compose_file"
            ;;
    esac
    
    success "$env_name environment setup completed"
}

deploy_to_environment() {
    local env_name="$1"
    local env_link="$FOXHUNT_HOME/$env_name"
    local compose_file="$env_link/docker-compose.${env_name}.yml"
    
    log "Deploying services to $env_name environment..."
    
    # Stop existing services in this environment
    if [ -f "$compose_file" ]; then
        cd "$env_link"
        docker-compose -f "docker-compose.${env_name}.yml" down --remove-orphans || true
    fi
    
    # Start new services
    cd "$env_link"
    docker-compose -f "docker-compose.${env_name}.yml" up -d
    
    # Wait for services to start
    sleep 15
    
    # Validate each service
    case $env_name in
        blue)
            local base_port=8080
            ;;
        green)
            local base_port=8180
            ;;
    esac
    
    for i in "${!SERVICES[@]}"; do
        local service="${SERVICES[$i]}"
        local port=$((base_port + i))
        local health_url="http://localhost:${port}/health"
        
        if ! health_check "$service-$env_name" "$health_url"; then
            error "Service $service failed in $env_name environment"
            return 1
        fi
    done
    
    success "All services deployed successfully to $env_name environment"
}

switch_traffic() {
    local new_env="$1"
    local env_link="$FOXHUNT_HOME/$new_env"
    
    log "Switching traffic to $new_env environment..."
    
    # Update load balancer configuration
    case $new_env in
        blue)
            local upstream_port_base=8080
            ;;
        green)
            local upstream_port_base=8180
            ;;
    esac
    
    # Generate new nginx upstream configuration
    cat > /tmp/foxhunt-upstream.conf << EOF
upstream foxhunt_core {
    server 127.0.0.1:${upstream_port_base};
}

upstream foxhunt_tli {
    server 127.0.0.1:$((upstream_port_base + 1));
}

upstream foxhunt_ml {
    server 127.0.0.1:$((upstream_port_base + 2));
}

upstream foxhunt_risk {
    server 127.0.0.1:$((upstream_port_base + 3));
}

upstream foxhunt_data {
    server 127.0.0.1:$((upstream_port_base + 4));
}
EOF
    
    # Update nginx configuration atomically
    sudo cp /tmp/foxhunt-upstream.conf /etc/nginx/conf.d/foxhunt-upstream.conf
    sudo nginx -t
    
    if [ $? -eq 0 ]; then
        sudo systemctl reload nginx
        
        # Update current symlink
        ln -sfn "$env_link" "$CURRENT_LINK"
        
        success "Traffic switched to $new_env environment"
    else
        error "Nginx configuration test failed"
        return 1
    fi
}

cleanup_old_environment() {
    local old_env="$1"
    local env_link="$FOXHUNT_HOME/$old_env"
    local compose_file="$env_link/docker-compose.${old_env}.yml"
    
    log "Cleaning up $old_env environment..."
    
    if [ -f "$compose_file" ]; then
        cd "$env_link"
        docker-compose -f "docker-compose.${old_env}.yml" down --remove-orphans
        docker-compose -f "docker-compose.${old_env}.yml" rm -f
    fi
    
    success "$old_env environment cleaned up"
}

blue_green_deployment() {
    local new_version="$1"
    local validate_only="$2"
    local skip_performance="$3"
    local keep_old="$4"
    
    local release_dir="$RELEASES_DIR/$new_version"
    
    log "Starting blue-green deployment for version $new_version..."
    
    # Validate release exists
    if [ ! -d "$release_dir" ]; then
        error "Release directory not found: $release_dir"
        return 1
    fi
    
    # Determine current and target environments
    local current_env
    current_env=$(determine_current_environment)
    local target_env
    
    if [ "$current_env" == "blue" ]; then
        target_env="green"
    else
        target_env="blue"
    fi
    
    info "Current environment: $current_env"
    info "Target environment: $target_env"
    
    if [ "$validate_only" == "true" ]; then
        log "Validation-only mode: would deploy version $new_version to $target_env"
        return 0
    fi
    
    # Setup target environment
    setup_environment "$target_env" "$new_version"
    
    # Deploy to target environment
    if ! deploy_to_environment "$target_env"; then
        error "Deployment to $target_env failed"
        return 1
    fi
    
    # Performance validation
    if [ "$skip_performance" != "true" ]; then
        log "Running performance validation on $target_env..."
        
        case $target_env in
            blue) local perf_port=8080 ;;
            green) local perf_port=8180 ;;
        esac
        
        if ! validate_performance "foxhunt-core-$target_env" "http://localhost:$perf_port"; then
            error "Performance validation failed on $target_env"
            return 1
        fi
    fi
    
    # Switch traffic
    if ! switch_traffic "$target_env"; then
        error "Traffic switching failed"
        return 1
    fi
    
    # Cleanup old environment unless requested to keep
    if [ "$keep_old" != "true" ]; then
        cleanup_old_environment "$current_env"
    fi
    
    # Record successful deployment
    echo "$new_version" > "$FOXHUNT_HOME/.blue-green-version"
    echo "$target_env" > "$FOXHUNT_HOME/.active-environment"
    
    success "Blue-green deployment completed successfully"
    success "Version $new_version is now active in $target_env environment"
    
    return 0
}

main() {
    local version=""
    local validate_only=false
    local skip_performance=false
    local keep_old=false
    
    # Parse arguments
    while [[ $# -gt 0 ]]; do
        case $1 in
            --validate-only)
                validate_only=true
                shift
                ;;
            --skip-performance)
                skip_performance=true
                shift
                ;;
            --keep-old)
                keep_old=true
                shift
                ;;
            -h|--help)
                usage
                ;;
            *)
                if [ -z "$version" ]; then
                    version="$1"
                else
                    error "Unknown option: $1"
                    usage
                fi
                shift
                ;;
        esac
    done
    
    # Validate version provided
    if [ -z "$version" ] && [ "$validate_only" = false ]; then
        error "Version is required for deployment"
        usage
    fi
    
    log "Starting blue-green deployment script..."
    log "Version: $version"
    log "Validate only: $validate_only"
    log "Skip performance: $skip_performance"
    log "Keep old: $keep_old"
    
    # Execute blue-green deployment
    blue_green_deployment "$version" "$validate_only" "$skip_performance" "$keep_old"
    
    if [ $? -eq 0 ]; then
        success "Blue-green deployment completed successfully"
        return 0
    else
        error "Blue-green deployment failed"
        return 1
    fi
}

# Execute main function
main "$@"