#!/bin/bash # Foxhunt Production Deployment Script # Wave 5 W5-4: Zero-downtime Kubernetes deployment with rolling updates # # Usage: ./scripts/deployment/deploy.sh [environment] # environment: dev|staging|prod (default: staging) set -euo pipefail # Color output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' # No Color # Configuration ENVIRONMENT="${1:-staging}" NAMESPACE="foxhunt" KUBE_CONTEXT="${KUBE_CONTEXT:-foxhunt-${ENVIRONMENT}}" TIMEOUT="${TIMEOUT:-600}" # 10 minutes HEALTH_CHECK_RETRIES=30 HEALTH_CHECK_INTERVAL=10 # Service list SERVICES=( "api-gateway" "trading-service" "backtesting-service" "ml-training-service" "trading-agent-service" ) echo -e "${GREEN}========================================${NC}" echo -e "${GREEN}Foxhunt Production Deployment${NC}" echo -e "${GREEN}Environment: ${ENVIRONMENT}${NC}" echo -e "${GREEN}Namespace: ${NAMESPACE}${NC}" echo -e "${GREEN}========================================${NC}" # Function: Check prerequisites check_prerequisites() { echo -e "\n${YELLOW}[1/6] Checking prerequisites...${NC}" # Check kubectl if ! command -v kubectl &> /dev/null; then echo -e "${RED}Error: kubectl not found. Please install kubectl.${NC}" exit 1 fi # Check kubeval if ! command -v kubeval &> /dev/null; then echo -e "${YELLOW}Warning: kubeval not found. Skipping manifest validation.${NC}" fi # Check context if ! kubectl config use-context "${KUBE_CONTEXT}" &> /dev/null; then echo -e "${RED}Error: Cannot switch to context ${KUBE_CONTEXT}${NC}" exit 1 fi echo -e "${GREEN}✓ Prerequisites check passed${NC}" } # Function: Validate manifests validate_manifests() { echo -e "\n${YELLOW}[2/6] Validating Kubernetes manifests...${NC}" if command -v kubeval &> /dev/null; then if kubeval k8s/deployments/*.yaml k8s/foxhunt-complete.yaml; then echo -e "${GREEN}✓ All manifests are valid${NC}" else echo -e "${RED}Error: Manifest validation failed${NC}" exit 1 fi else echo -e "${YELLOW}Skipping validation (kubeval not installed)${NC}" fi } # Function: Create or update namespace and base resources deploy_base_resources() { echo -e "\n${YELLOW}[3/6] Deploying base resources...${NC}" # Create namespace if it doesn't exist kubectl create namespace "${NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f - # Apply ConfigMaps, Secrets, and PVCs kubectl apply -f k8s/foxhunt-complete.yaml --namespace="${NAMESPACE}" # Wait for PVCs to be bound echo "Waiting for PVCs to be bound..." kubectl wait --for=condition=Bound \ pvc/postgres-pvc \ pvc/redis-pvc \ pvc/prometheus-pvc \ pvc/ml-models-pvc \ pvc/ml-checkpoints-pvc \ pvc/backtesting-data-pvc \ pvc/backtesting-results-pvc \ --namespace="${NAMESPACE}" \ --timeout="${TIMEOUT}s" || true echo -e "${GREEN}✓ Base resources deployed${NC}" } # Function: Deploy services with rolling updates deploy_services() { echo -e "\n${YELLOW}[4/6] Deploying services with rolling updates...${NC}" for service in "${SERVICES[@]}"; do echo -e "\n${YELLOW}Deploying ${service}...${NC}" # Apply deployment kubectl apply -f "k8s/deployments/${service}-deployment.yaml" --namespace="${NAMESPACE}" # Apply service if [ -f "k8s/services/${service}-service.yaml" ]; then kubectl apply -f "k8s/services/${service}-service.yaml" --namespace="${NAMESPACE}" fi # Wait for rollout echo "Waiting for ${service} rollout to complete..." if kubectl rollout status deployment/"${service}" \ --namespace="${NAMESPACE}" \ --timeout="${TIMEOUT}s"; then echo -e "${GREEN}✓ ${service} deployed successfully${NC}" else echo -e "${RED}Error: ${service} deployment failed${NC}" # Show pod logs for debugging echo "Recent pod logs:" kubectl logs -l app="${service}" --namespace="${NAMESPACE}" --tail=50 || true # Ask if we should continue read -p "Continue with deployment? (y/N) " -n 1 -r echo if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1 fi fi done echo -e "${GREEN}✓ All services deployed${NC}" } # Function: Apply HPAs deploy_autoscaling() { echo -e "\n${YELLOW}[5/6] Deploying HorizontalPodAutoscalers...${NC}" kubectl apply -f k8s/foxhunt-complete.yaml --namespace="${NAMESPACE}" echo -e "${GREEN}✓ Autoscaling configured${NC}" } # Function: Health checks run_health_checks() { echo -e "\n${YELLOW}[6/6] Running health checks...${NC}" for service in "${SERVICES[@]}"; do echo -e "\n${YELLOW}Checking ${service} health...${NC}" # Get pod name POD=$(kubectl get pods -l app="${service}" \ --namespace="${NAMESPACE}" \ -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") if [ -z "$POD" ]; then echo -e "${RED}✗ No pods found for ${service}${NC}" continue fi # Get health port based on service case $service in api-gateway) HEALTH_PORT=8080 HEALTH_PATH="/health/liveness" ;; trading-service) HEALTH_PORT=8081 HEALTH_PATH="/health" ;; backtesting-service) HEALTH_PORT=8082 HEALTH_PATH="/health" ;; ml-training-service) HEALTH_PORT=8095 HEALTH_PATH="/health" ;; trading-agent-service) HEALTH_PORT=8083 HEALTH_PATH="/health" ;; esac # Check health endpoint for i in $(seq 1 $HEALTH_CHECK_RETRIES); do if kubectl exec -it "$POD" --namespace="${NAMESPACE}" -- \ curl -f "http://localhost:${HEALTH_PORT}${HEALTH_PATH}" &> /dev/null; then echo -e "${GREEN}✓ ${service} is healthy${NC}" break else if [ $i -eq $HEALTH_CHECK_RETRIES ]; then echo -e "${RED}✗ ${service} health check failed after ${HEALTH_CHECK_RETRIES} attempts${NC}" else echo "Health check attempt $i/${HEALTH_CHECK_RETRIES} failed, retrying in ${HEALTH_CHECK_INTERVAL}s..." sleep $HEALTH_CHECK_INTERVAL fi fi done done echo -e "\n${GREEN}✓ Health checks completed${NC}" } # Function: Show deployment summary show_summary() { echo -e "\n${GREEN}========================================${NC}" echo -e "${GREEN}Deployment Summary${NC}" echo -e "${GREEN}========================================${NC}" echo -e "\nPods:" kubectl get pods --namespace="${NAMESPACE}" -o wide echo -e "\nServices:" kubectl get svc --namespace="${NAMESPACE}" echo -e "\nHPAs:" kubectl get hpa --namespace="${NAMESPACE}" echo -e "\n${GREEN}Deployment completed successfully!${NC}" echo -e "Access the API Gateway at: $(kubectl get svc api-gateway --namespace="${NAMESPACE}" -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):50051" } # Main deployment flow main() { check_prerequisites validate_manifests deploy_base_resources deploy_services deploy_autoscaling run_health_checks show_summary } # Run main function main