#!/bin/bash # Blue-Green Deployment Script for Foxhunt HFT Platform # Implements zero-downtime deployment with shadow traffic validation set -euo pipefail # Configuration NAMESPACE="foxhunt-production" ARGOCD_NAMESPACE="argocd" SHADOW_TRAFFIC_PERCENTAGE=5 VALIDATION_DURATION=300 LATENCY_THRESHOLD=50 THROUGHPUT_THRESHOLD=100000 # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color # Logging functions log_info() { echo -e "${BLUE}[INFO]${NC} $1" } log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1" } log_warning() { echo -e "${YELLOW}[WARNING]${NC} $1" } log_error() { echo -e "${RED}[ERROR]${NC} $1" } # Function to check if a command exists command_exists() { command -v "$1" >/dev/null 2>&1 } # Validate prerequisites validate_prerequisites() { log_info "Validating prerequisites..." if ! command_exists kubectl; then log_error "kubectl is not installed" exit 1 fi if ! command_exists jq; then log_error "jq is not installed" exit 1 fi if ! kubectl auth can-i get applications -n "$ARGOCD_NAMESPACE" >/dev/null 2>&1; then log_error "Insufficient permissions to access ArgoCD applications" exit 1 fi log_success "Prerequisites validated" } # Get current active slot get_current_slot() { local current_slot current_slot=$(kubectl get service foxhunt-platform-active \ -n "$NAMESPACE" \ -o jsonpath='{.spec.selector.slot}' 2>/dev/null || echo "blue") echo "$current_slot" } # Get target slot for deployment get_target_slot() { local current_slot="$1" if [ "$current_slot" = "blue" ]; then echo "green" else echo "blue" fi } # Wait for ArgoCD application to be healthy wait_for_application_health() { local app_name="$1" local timeout="$2" log_info "Waiting for application $app_name to be healthy (timeout: ${timeout}s)..." if kubectl wait --for=condition=Healthy \ "application/$app_name" \ -n "$ARGOCD_NAMESPACE" \ --timeout="${timeout}s" >/dev/null 2>&1; then log_success "Application $app_name is healthy" return 0 else log_error "Application $app_name failed to become healthy within ${timeout}s" return 1 fi } # Deploy to target slot deploy_to_slot() { local slot="$1" local image_tag="$2" local app_name="foxhunt-platform-$slot" log_info "Deploying to $slot slot with image tag: $image_tag" # Update the ArgoCD application with new image tag kubectl patch application "$app_name" \ -n "$ARGOCD_NAMESPACE" \ --type merge \ --patch "{\"spec\":{\"source\":{\"helm\":{\"parameters\":[{\"name\":\"global.imageTag\",\"value\":\"$image_tag\"}]}}}}" # Trigger sync kubectl patch application "$app_name" \ -n "$ARGOCD_NAMESPACE" \ --type merge \ --patch '{"operation":{"sync":{}}}' # Wait for deployment to complete if ! wait_for_application_health "$app_name" 900; then log_error "Deployment to $slot slot failed" return 1 fi log_success "Deployment to $slot slot completed" } # Configure shadow traffic configure_shadow_traffic() { local target_slot="$1" local percentage="$2" log_info "Configuring $percentage% shadow traffic to $target_slot slot" # Apply shadow traffic configuration cat < /tmp/perf_test.py << 'EOF' import time import requests import statistics import sys import threading from concurrent.futures import ThreadPoolExecutor, as_completed def measure_latency(url, duration): """Measure latency for the specified duration""" latencies = [] errors = 0 start_time = time.time() while time.time() - start_time < duration: try: start = time.time() response = requests.get(f"{url}/health", timeout=1) end = time.time() if response.status_code == 200: latency_ms = (end - start) * 1000 latencies.append(latency_ms) else: errors += 1 except Exception: errors += 1 time.sleep(0.001) # 1ms between requests return latencies, errors def main(): slot = sys.argv[1] duration = int(sys.argv[2]) latency_threshold = float(sys.argv[3]) url = f"http://foxhunt-platform-{slot}.foxhunt-production.svc.cluster.local:8080" latencies, errors = measure_latency(url, duration) if not latencies: print(f"ERROR: No successful requests during {duration}s test") sys.exit(1) avg_latency = statistics.mean(latencies) p95_latency = statistics.quantiles(latencies, n=20)[18] # 95th percentile p99_latency = statistics.quantiles(latencies, n=100)[98] # 99th percentile print(f"Performance Results for {slot} slot:") print(f" Total requests: {len(latencies)}") print(f" Errors: {errors}") print(f" Average latency: {avg_latency:.2f}ms") print(f" 95th percentile: {p95_latency:.2f}ms") print(f" 99th percentile: {p99_latency:.2f}ms") # Check if performance meets requirements if p95_latency > latency_threshold: print(f"ERROR: 95th percentile latency ({p95_latency:.2f}ms) exceeds threshold ({latency_threshold}ms)") sys.exit(1) if errors > len(latencies) * 0.001: # More than 0.1% error rate print(f"ERROR: Error rate ({errors}/{len(latencies)}) exceeds threshold") sys.exit(1) print("Performance validation PASSED") if __name__ == "__main__": main() EOF # Run performance test if python3 /tmp/perf_test.py "$slot" "$duration" "$LATENCY_THRESHOLD"; then log_success "Performance validation passed for $slot slot" rm -f /tmp/perf_test.py return 0 else log_error "Performance validation failed for $slot slot" rm -f /tmp/perf_test.py return 1 fi } # Switch traffic to new slot switch_traffic() { local target_slot="$1" log_info "Switching active traffic to $target_slot slot" # Update the active service selector kubectl patch service foxhunt-platform-active \ -n "$NAMESPACE" \ --type merge \ --patch "{\"spec\":{\"selector\":{\"slot\":\"$target_slot\"}}}" log_success "Traffic switched to $target_slot slot" } # Rollback to previous slot rollback() { local current_slot="$1" local previous_slot previous_slot=$(get_target_slot "$current_slot") log_warning "Initiating emergency rollback to $previous_slot slot" # Switch traffic back kubectl patch service foxhunt-platform-active \ -n "$NAMESPACE" \ --type merge \ --patch "{\"spec\":{\"selector\":{\"slot\":\"$previous_slot\"}}}" # Remove shadow traffic configuration kubectl delete virtualservice foxhunt-platform-shadow -n "$NAMESPACE" --ignore-not-found=true log_success "Emergency rollback to $previous_slot completed" } # Cleanup old slot cleanup_old_slot() { local old_slot="$1" log_info "Scaling down $old_slot slot" # Scale down the old slot kubectl patch application "foxhunt-platform-$old_slot" \ -n "$ARGOCD_NAMESPACE" \ --type merge \ --patch '{"spec":{"source":{"helm":{"parameters":[{"name":"global.replicaCount","value":"0"}]}}}}' log_success "Old slot $old_slot scaled down" } # Main deployment function deploy() { local image_tag="$1" log_info "Starting blue-green deployment with image tag: $image_tag" # Validate prerequisites validate_prerequisites # Determine deployment slots local current_slot local target_slot current_slot=$(get_current_slot) target_slot=$(get_target_slot "$current_slot") log_info "Current active slot: $current_slot" log_info "Target deployment slot: $target_slot" # Deploy to target slot if ! deploy_to_slot "$target_slot" "$image_tag"; then log_error "Deployment failed" exit 1 fi # Configure shadow traffic for validation configure_shadow_traffic "$target_slot" "$SHADOW_TRAFFIC_PERCENTAGE" # Wait for shadow traffic to stabilize log_info "Waiting for shadow traffic to stabilize..." sleep 30 # Validate performance with shadow traffic if ! validate_performance "$target_slot" "$VALIDATION_DURATION"; then log_error "Performance validation failed with shadow traffic" rollback "$target_slot" exit 1 fi # Switch traffic to new slot switch_traffic "$target_slot" # Post-deployment validation log_info "Running post-deployment validation..." sleep 60 if ! validate_performance "$target_slot" 180; then log_error "Post-deployment validation failed" rollback "$target_slot" exit 1 fi # Cleanup shadow traffic configuration kubectl delete virtualservice foxhunt-platform-shadow -n "$NAMESPACE" --ignore-not-found=true # Cleanup old slot after successful deployment cleanup_old_slot "$current_slot" log_success "Blue-green deployment completed successfully!" log_info "Active slot is now: $target_slot" } # Script usage usage() { echo "Usage: $0 " echo "" echo "Arguments:" echo " image_tag The container image tag to deploy" echo "" echo "Environment variables:" echo " NAMESPACE Kubernetes namespace (default: foxhunt-production)" echo " SHADOW_TRAFFIC_PERCENTAGE Shadow traffic percentage (default: 5)" echo " VALIDATION_DURATION Validation duration in seconds (default: 300)" echo " LATENCY_THRESHOLD Maximum allowed latency in ms (default: 50)" echo "" echo "Example:" echo " $0 v1.2.3" echo " $0 \$GITHUB_SHA" } # Main script execution main() { if [ $# -ne 1 ]; then usage exit 1 fi local image_tag="$1" # Validate image tag format if [[ ! "$image_tag" =~ ^[a-zA-Z0-9._-]+$ ]]; then log_error "Invalid image tag format: $image_tag" exit 1 fi # Set trap for cleanup on script exit trap 'log_info "Deployment script interrupted"' INT TERM # Execute deployment deploy "$image_tag" } # Execute main function with all arguments main "$@"