#!/bin/bash # Foxhunt Deployment Rollback Script # Wave 5 W5-4: Safe rollback to previous deployment revision # # Usage: ./scripts/deployment/rollback.sh [service] [revision] # service: api-gateway|trading-service|backtesting-service|ml-training-service|trading-agent-service|all # revision: revision number (optional, defaults to previous) set -euo pipefail # Color output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' # Configuration SERVICE="${1:-all}" REVISION="${2:-0}" # 0 = previous revision NAMESPACE="foxhunt" TIMEOUT=600 # Service list ALL_SERVICES=( "api-gateway" "trading-service" "backtesting-service" "ml-training-service" "trading-agent-service" ) echo -e "${YELLOW}========================================${NC}" echo -e "${YELLOW}Foxhunt Deployment Rollback${NC}" echo -e "${YELLOW}Service: ${SERVICE}${NC}" echo -e "${YELLOW}Namespace: ${NAMESPACE}${NC}" echo -e "${YELLOW}========================================${NC}" # Function: Show deployment history show_history() { local service=$1 echo -e "\n${YELLOW}Deployment history for ${service}:${NC}" kubectl rollout history deployment/"${service}" --namespace="${NAMESPACE}" || true } # Function: Rollback a service rollback_service() { local service=$1 local revision=$2 echo -e "\n${YELLOW}Rolling back ${service}...${NC}" # Show current history show_history "${service}" # Confirm rollback if [ "${REVISION}" == "0" ]; then echo -e "${RED}This will rollback ${service} to the previous revision.${NC}" else echo -e "${RED}This will rollback ${service} to revision ${revision}.${NC}" fi read -p "Are you sure? (y/N) " -n 1 -r echo if [[ ! $REPLY =~ ^[Yy]$ ]]; then echo "Rollback cancelled." return fi # Perform rollback if [ "${revision}" == "0" ]; then kubectl rollout undo deployment/"${service}" --namespace="${NAMESPACE}" else kubectl rollout undo deployment/"${service}" --namespace="${NAMESPACE}" --to-revision="${revision}" fi # Wait for rollout echo "Waiting for rollback to complete..." if kubectl rollout status deployment/"${service}" \ --namespace="${NAMESPACE}" \ --timeout="${TIMEOUT}s"; then echo -e "${GREEN}✓ ${service} rolled back successfully${NC}" else echo -e "${RED}✗ ${service} rollback failed${NC}" # Show pod logs echo "Recent pod logs:" kubectl logs -l app="${service}" --namespace="${NAMESPACE}" --tail=50 || true # Show events echo -e "\nRecent events:" kubectl get events --namespace="${NAMESPACE}" --field-selector involvedObject.name="${service}" --sort-by='.lastTimestamp' | tail -20 exit 1 fi # Verify health verify_health "${service}" } # Function: Verify service health after rollback verify_health() { local service=$1 echo -e "\n${YELLOW}Verifying ${service} health...${NC}" # Get pod name POD=$(kubectl get pods -l app="${service}" \ --namespace="${NAMESPACE}" \ -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") if [ -z "$POD" ]; then echo -e "${RED}✗ No pods found for ${service}${NC}" return 1 fi # Get health port case $service in api-gateway) HEALTH_PORT=8080 HEALTH_PATH="/health/liveness" ;; trading-service) HEALTH_PORT=8081 HEALTH_PATH="/health" ;; backtesting-service) HEALTH_PORT=8082 HEALTH_PATH="/health" ;; ml-training-service) HEALTH_PORT=8095 HEALTH_PATH="/health" ;; trading-agent-service) HEALTH_PORT=8083 HEALTH_PATH="/health" ;; esac # Check health sleep 5 # Give service time to stabilize if kubectl exec -it "$POD" --namespace="${NAMESPACE}" -- \ curl -f "http://localhost:${HEALTH_PORT}${HEALTH_PATH}" &> /dev/null; then echo -e "${GREEN}✓ ${service} is healthy after rollback${NC}" else echo -e "${RED}✗ ${service} health check failed after rollback${NC}" return 1 fi } # Function: Rollback all services rollback_all() { echo -e "\n${RED}WARNING: This will rollback ALL services!${NC}" read -p "Are you ABSOLUTELY sure? Type 'YES' to confirm: " -r echo if [[ ! $REPLY == "YES" ]]; then echo "Rollback cancelled." exit 0 fi # Rollback in reverse order (agent -> ml -> backtesting -> trading -> gateway) for service in "${ALL_SERVICES[@]}"; do rollback_service "${service}" "${REVISION}" done echo -e "\n${GREEN}All services rolled back successfully${NC}" } # Function: Show summary show_summary() { echo -e "\n${GREEN}========================================${NC}" echo -e "${GREEN}Rollback Summary${NC}" echo -e "${GREEN}========================================${NC}" echo -e "\nPods:" kubectl get pods --namespace="${NAMESPACE}" -o wide echo -e "\nDeployments:" kubectl get deployments --namespace="${NAMESPACE}" echo -e "\n${GREEN}Rollback completed!${NC}" } # Main rollback flow main() { # Check kubectl if ! command -v kubectl &> /dev/null; then echo -e "${RED}Error: kubectl not found${NC}" exit 1 fi # Perform rollback if [ "${SERVICE}" == "all" ]; then rollback_all else # Validate service name if [[ ! " ${ALL_SERVICES[@]} " =~ " ${SERVICE} " ]]; then echo -e "${RED}Error: Invalid service name '${SERVICE}'${NC}" echo "Valid services: ${ALL_SERVICES[*]} or 'all'" exit 1 fi rollback_service "${SERVICE}" "${REVISION}" fi show_summary } # Run main function main