#!/bin/bash # Blue-green deployment script for Foxhunt HFT Trading System # Implements zero-downtime deployment with instant traffic switching set -euo pipefail # Configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DEPLOYMENT_LOG="/home/jgrusewski/Work/foxhunt/logs/blue-green-deployment-$(date +%s).log" FOXHUNT_HOME="/opt/foxhunt" RELEASES_DIR="/opt/foxhunt/releases" CURRENT_LINK="/opt/foxhunt/current" GREEN_LINK="/opt/foxhunt/green" BLUE_LINK="/opt/foxhunt/blue" LOAD_BALANCER_CONFIG="/etc/nginx/sites-available/foxhunt-lb" # Services in dependency order SERVICES=("foxhunt-core" "foxhunt-data" "foxhunt-risk" "foxhunt-ml" "foxhunt-tli") # Performance thresholds MAX_LATENCY_US=30 MIN_THROUGHPUT_OPS=1000 # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$DEPLOYMENT_LOG" } error() { echo -e "${RED}ERROR: $1${NC}" | tee -a "$DEPLOYMENT_LOG" } success() { echo -e "${GREEN}SUCCESS: $1${NC}" | tee -a "$DEPLOYMENT_LOG" } warning() { echo -e "${YELLOW}WARNING: $1${NC}" | tee -a "$DEPLOYMENT_LOG" } info() { echo -e "${BLUE}INFO: $1${NC}" | tee -a "$DEPLOYMENT_LOG" } cleanup() { log "Cleaning up blue-green deployment artifacts..." exit "${1:-1}" } trap cleanup EXIT INT TERM usage() { echo "Usage: $0 [options]" echo "Options:" echo " --validate-only Only validate deployment, don't deploy" echo " --skip-performance Skip performance validation" echo " --keep-old Keep old environment after successful deployment" exit 1 } validate_performance() { local service_name="$1" local endpoint="$2" log "Validating performance for $service_name..." # Check latency with multiple samples local total_latency=0 local samples=10 for i in $(seq 1 $samples); do local start_time=$(date +%s%N) if curl -f -s "$endpoint/health" > /dev/null 2>&1; then local end_time=$(date +%s%N) local latency_ns=$((end_time - start_time)) local latency_us=$((latency_ns / 1000)) total_latency=$((total_latency + latency_us)) else error "Health check failed for $service_name" return 1 fi sleep 0.1 done local avg_latency_us=$((total_latency / samples)) if [ "$avg_latency_us" -gt "$MAX_LATENCY_US" ]; then error "Average latency validation failed: ${avg_latency_us}μs > ${MAX_LATENCY_US}μs threshold" return 1 fi # Check system metrics local cpu_usage cpu_usage=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | cut -d'%' -f1) if (( $(echo "$cpu_usage > 80" | bc -l) )); then warning "High CPU usage detected: ${cpu_usage}%" fi success "Performance validation passed - Average latency: ${avg_latency_us}μs, CPU: ${cpu_usage}%" return 0 } health_check() { local service_name="$1" local health_url="$2" local max_attempts=30 local attempt=0 log "Health checking $service_name at $health_url..." while [ $attempt -lt $max_attempts ]; do if curl -f -s "$health_url" > /dev/null 2>&1; then # Additional gRPC check for TLI if [[ "$service_name" == *"tli"* ]]; then local grpc_port=$(echo "$health_url" | sed 's/.*:\([0-9]*\).*/\1/') local grpc_check_port=$((grpc_port + 1000)) # Assume gRPC is on different port if grpcurl -plaintext "localhost:$grpc_check_port" list > /dev/null 2>&1; then success "$service_name is healthy (HTTP + gRPC)" return 0 fi else success "$service_name is healthy" return 0 fi fi attempt=$((attempt + 1)) log "Health check attempt $attempt/$max_attempts failed, retrying in 3s..." sleep 3 done error "$service_name failed health check after $max_attempts attempts" return 1 } determine_current_environment() { log "Determining current active environment..." if [ -L "$CURRENT_LINK" ]; then local current_target current_target=$(readlink "$CURRENT_LINK") if [[ "$current_target" == *"blue"* ]]; then echo "blue" elif [[ "$current_target" == *"green"* ]]; then echo "green" else # Default to blue if unclear echo "blue" fi else # No current deployment, default to blue echo "blue" fi } setup_environment() { local env_name="$1" local version="$2" local release_dir="$RELEASES_DIR/$version" local env_link="$FOXHUNT_HOME/$env_name" log "Setting up $env_name environment with version $version..." # Create environment directory mkdir -p "$env_link" # Copy release to environment rsync -a --delete "$release_dir/" "$env_link/" # Update Docker Compose for this environment local compose_file="$env_link/docker-compose.${env_name}.yml" cp "$FOXHUNT_HOME/docker/docker-compose.production.yml" "$compose_file" # Update container names to include environment sed -i "s/foxhunt-\([^-]*\)-prod/foxhunt-\1-${env_name}/g" "$compose_file" # Update port mappings to avoid conflicts case $env_name in blue) # Blue uses standard ports (8080, 8081, etc.) ;; green) # Green uses offset ports (8180, 8181, etc.) sed -i 's/:808\([0-9]\)/:818\1/g' "$compose_file" sed -i 's/:909\([0-9]\)/:919\1/g' "$compose_file" sed -i 's/:5005\([0-9]\)/:5105\1/g' "$compose_file" ;; esac success "$env_name environment setup completed" } deploy_to_environment() { local env_name="$1" local env_link="$FOXHUNT_HOME/$env_name" local compose_file="$env_link/docker-compose.${env_name}.yml" log "Deploying services to $env_name environment..." # Stop existing services in this environment if [ -f "$compose_file" ]; then cd "$env_link" docker-compose -f "docker-compose.${env_name}.yml" down --remove-orphans || true fi # Start new services cd "$env_link" docker-compose -f "docker-compose.${env_name}.yml" up -d # Wait for services to start sleep 15 # Validate each service case $env_name in blue) local base_port=8080 ;; green) local base_port=8180 ;; esac for i in "${!SERVICES[@]}"; do local service="${SERVICES[$i]}" local port=$((base_port + i)) local health_url="http://localhost:${port}/health" if ! health_check "$service-$env_name" "$health_url"; then error "Service $service failed in $env_name environment" return 1 fi done success "All services deployed successfully to $env_name environment" } switch_traffic() { local new_env="$1" local env_link="$FOXHUNT_HOME/$new_env" log "Switching traffic to $new_env environment..." # Update load balancer configuration case $new_env in blue) local upstream_port_base=8080 ;; green) local upstream_port_base=8180 ;; esac # Generate new nginx upstream configuration cat > /tmp/foxhunt-upstream.conf << EOF upstream foxhunt_core { server 127.0.0.1:${upstream_port_base}; } upstream foxhunt_tli { server 127.0.0.1:$((upstream_port_base + 1)); } upstream foxhunt_ml { server 127.0.0.1:$((upstream_port_base + 2)); } upstream foxhunt_risk { server 127.0.0.1:$((upstream_port_base + 3)); } upstream foxhunt_data { server 127.0.0.1:$((upstream_port_base + 4)); } EOF # Update nginx configuration atomically sudo cp /tmp/foxhunt-upstream.conf /etc/nginx/conf.d/foxhunt-upstream.conf sudo nginx -t if [ $? -eq 0 ]; then sudo systemctl reload nginx # Update current symlink ln -sfn "$env_link" "$CURRENT_LINK" success "Traffic switched to $new_env environment" else error "Nginx configuration test failed" return 1 fi } cleanup_old_environment() { local old_env="$1" local env_link="$FOXHUNT_HOME/$old_env" local compose_file="$env_link/docker-compose.${old_env}.yml" log "Cleaning up $old_env environment..." if [ -f "$compose_file" ]; then cd "$env_link" docker-compose -f "docker-compose.${old_env}.yml" down --remove-orphans docker-compose -f "docker-compose.${old_env}.yml" rm -f fi success "$old_env environment cleaned up" } blue_green_deployment() { local new_version="$1" local validate_only="$2" local skip_performance="$3" local keep_old="$4" local release_dir="$RELEASES_DIR/$new_version" log "Starting blue-green deployment for version $new_version..." # Validate release exists if [ ! -d "$release_dir" ]; then error "Release directory not found: $release_dir" return 1 fi # Determine current and target environments local current_env current_env=$(determine_current_environment) local target_env if [ "$current_env" == "blue" ]; then target_env="green" else target_env="blue" fi info "Current environment: $current_env" info "Target environment: $target_env" if [ "$validate_only" == "true" ]; then log "Validation-only mode: would deploy version $new_version to $target_env" return 0 fi # Setup target environment setup_environment "$target_env" "$new_version" # Deploy to target environment if ! deploy_to_environment "$target_env"; then error "Deployment to $target_env failed" return 1 fi # Performance validation if [ "$skip_performance" != "true" ]; then log "Running performance validation on $target_env..." case $target_env in blue) local perf_port=8080 ;; green) local perf_port=8180 ;; esac if ! validate_performance "foxhunt-core-$target_env" "http://localhost:$perf_port"; then error "Performance validation failed on $target_env" return 1 fi fi # Switch traffic if ! switch_traffic "$target_env"; then error "Traffic switching failed" return 1 fi # Cleanup old environment unless requested to keep if [ "$keep_old" != "true" ]; then cleanup_old_environment "$current_env" fi # Record successful deployment echo "$new_version" > "$FOXHUNT_HOME/.blue-green-version" echo "$target_env" > "$FOXHUNT_HOME/.active-environment" success "Blue-green deployment completed successfully" success "Version $new_version is now active in $target_env environment" return 0 } main() { local version="" local validate_only=false local skip_performance=false local keep_old=false # Parse arguments while [[ $# -gt 0 ]]; do case $1 in --validate-only) validate_only=true shift ;; --skip-performance) skip_performance=true shift ;; --keep-old) keep_old=true shift ;; -h|--help) usage ;; *) if [ -z "$version" ]; then version="$1" else error "Unknown option: $1" usage fi shift ;; esac done # Validate version provided if [ -z "$version" ] && [ "$validate_only" = false ]; then error "Version is required for deployment" usage fi log "Starting blue-green deployment script..." log "Version: $version" log "Validate only: $validate_only" log "Skip performance: $skip_performance" log "Keep old: $keep_old" # Execute blue-green deployment blue_green_deployment "$version" "$validate_only" "$skip_performance" "$keep_old" if [ $? -eq 0 ]; then success "Blue-green deployment completed successfully" return 0 else error "Blue-green deployment failed" return 1 fi } # Execute main function main "$@"