foxhunt/scripts/deployment/deploy.sh

#!/bin/bash
# Foxhunt Production Deployment Script
# Wave 5 W5-4: Zero-downtime Kubernetes deployment with rolling updates
#
# Usage: ./scripts/deployment/deploy.sh [environment]
#   environment: dev|staging|prod (default: staging)

set -euo pipefail

# Color output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

# Configuration
ENVIRONMENT="${1:-staging}"
NAMESPACE="foxhunt"
KUBE_CONTEXT="${KUBE_CONTEXT:-foxhunt-${ENVIRONMENT}}"
TIMEOUT="${TIMEOUT:-600}"  # 10 minutes
HEALTH_CHECK_RETRIES=30
HEALTH_CHECK_INTERVAL=10

# Service list
SERVICES=(
    "api-gateway"
    "trading-service"
    "backtesting-service"
    "ml-training-service"
    "trading-agent-service"
)

echo -e "${GREEN}========================================${NC}"
echo -e "${GREEN}Foxhunt Production Deployment${NC}"
echo -e "${GREEN}Environment: ${ENVIRONMENT}${NC}"
echo -e "${GREEN}Namespace: ${NAMESPACE}${NC}"
echo -e "${GREEN}========================================${NC}"

# Function: Check prerequisites
check_prerequisites() {
    echo -e "\n${YELLOW}[1/6] Checking prerequisites...${NC}"

    # Check kubectl
    if ! command -v kubectl &> /dev/null; then
        echo -e "${RED}Error: kubectl not found. Please install kubectl.${NC}"
        exit 1
    fi

    # Check kubeval
    if ! command -v kubeval &> /dev/null; then
        echo -e "${YELLOW}Warning: kubeval not found. Skipping manifest validation.${NC}"
    fi

    # Check context
    if ! kubectl config use-context "${KUBE_CONTEXT}" &> /dev/null; then
        echo -e "${RED}Error: Cannot switch to context ${KUBE_CONTEXT}${NC}"
        exit 1
    fi

    echo -e "${GREEN}✓ Prerequisites check passed${NC}"
}

# Function: Validate manifests
validate_manifests() {
    echo -e "\n${YELLOW}[2/6] Validating Kubernetes manifests...${NC}"

    if command -v kubeval &> /dev/null; then
        if kubeval k8s/deployments/*.yaml k8s/foxhunt-complete.yaml; then
            echo -e "${GREEN}✓ All manifests are valid${NC}"
        else
            echo -e "${RED}Error: Manifest validation failed${NC}"
            exit 1
        fi
    else
        echo -e "${YELLOW}Skipping validation (kubeval not installed)${NC}"
    fi
}

# Function: Create or update namespace and base resources
deploy_base_resources() {
    echo -e "\n${YELLOW}[3/6] Deploying base resources...${NC}"

    # Create namespace if it doesn't exist
    kubectl create namespace "${NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f -

    # Apply ConfigMaps, Secrets, and PVCs
    kubectl apply -f k8s/foxhunt-complete.yaml --namespace="${NAMESPACE}"

    # Wait for PVCs to be bound
    echo "Waiting for PVCs to be bound..."
    kubectl wait --for=condition=Bound \
        pvc/postgres-pvc \
        pvc/redis-pvc \
        pvc/prometheus-pvc \
        pvc/ml-models-pvc \
        pvc/ml-checkpoints-pvc \
        pvc/backtesting-data-pvc \
        pvc/backtesting-results-pvc \
        --namespace="${NAMESPACE}" \
        --timeout="${TIMEOUT}s" || true

    echo -e "${GREEN}✓ Base resources deployed${NC}"
}

# Function: Deploy services with rolling updates
deploy_services() {
    echo -e "\n${YELLOW}[4/6] Deploying services with rolling updates...${NC}"

    for service in "${SERVICES[@]}"; do
        echo -e "\n${YELLOW}Deploying ${service}...${NC}"

        # Apply deployment
        kubectl apply -f "k8s/deployments/${service}-deployment.yaml" --namespace="${NAMESPACE}"

        # Apply service
        if [ -f "k8s/services/${service}-service.yaml" ]; then
            kubectl apply -f "k8s/services/${service}-service.yaml" --namespace="${NAMESPACE}"
        fi

        # Wait for rollout
        echo "Waiting for ${service} rollout to complete..."
        if kubectl rollout status deployment/"${service}" \
            --namespace="${NAMESPACE}" \
            --timeout="${TIMEOUT}s"; then
            echo -e "${GREEN}✓ ${service} deployed successfully${NC}"
        else
            echo -e "${RED}Error: ${service} deployment failed${NC}"

            # Show pod logs for debugging
            echo "Recent pod logs:"
            kubectl logs -l app="${service}" --namespace="${NAMESPACE}" --tail=50 || true

            # Ask if we should continue
            read -p "Continue with deployment? (y/N) " -n 1 -r
            echo
            if [[ ! $REPLY =~ ^[Yy]$ ]]; then
                exit 1
            fi
        fi
    done

    echo -e "${GREEN}✓ All services deployed${NC}"
}

# Function: Apply HPAs
deploy_autoscaling() {
    echo -e "\n${YELLOW}[5/6] Deploying HorizontalPodAutoscalers...${NC}"

    kubectl apply -f k8s/foxhunt-complete.yaml --namespace="${NAMESPACE}"

    echo -e "${GREEN}✓ Autoscaling configured${NC}"
}

# Function: Health checks
run_health_checks() {
    echo -e "\n${YELLOW}[6/6] Running health checks...${NC}"

    for service in "${SERVICES[@]}"; do
        echo -e "\n${YELLOW}Checking ${service} health...${NC}"

        # Get pod name
        POD=$(kubectl get pods -l app="${service}" \
            --namespace="${NAMESPACE}" \
            -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")

        if [ -z "$POD" ]; then
            echo -e "${RED}✗ No pods found for ${service}${NC}"
            continue
        fi

        # Get health port based on service
        case $service in
            api-gateway)
                HEALTH_PORT=8080
                HEALTH_PATH="/health/liveness"
                ;;
            trading-service)
                HEALTH_PORT=8081
                HEALTH_PATH="/health"
                ;;
            backtesting-service)
                HEALTH_PORT=8082
                HEALTH_PATH="/health"
                ;;
            ml-training-service)
                HEALTH_PORT=8095
                HEALTH_PATH="/health"
                ;;
            trading-agent-service)
                HEALTH_PORT=8083
                HEALTH_PATH="/health"
                ;;
        esac

        # Check health endpoint
        for i in $(seq 1 $HEALTH_CHECK_RETRIES); do
            if kubectl exec -it "$POD" --namespace="${NAMESPACE}" -- \
                curl -f "http://localhost:${HEALTH_PORT}${HEALTH_PATH}" &> /dev/null; then
                echo -e "${GREEN}✓ ${service} is healthy${NC}"
                break
            else
                if [ $i -eq $HEALTH_CHECK_RETRIES ]; then
                    echo -e "${RED}✗ ${service} health check failed after ${HEALTH_CHECK_RETRIES} attempts${NC}"
                else
                    echo "Health check attempt $i/${HEALTH_CHECK_RETRIES} failed, retrying in ${HEALTH_CHECK_INTERVAL}s..."
                    sleep $HEALTH_CHECK_INTERVAL
                fi
            fi
        done
    done

    echo -e "\n${GREEN}✓ Health checks completed${NC}"
}

# Function: Show deployment summary
show_summary() {
    echo -e "\n${GREEN}========================================${NC}"
    echo -e "${GREEN}Deployment Summary${NC}"
    echo -e "${GREEN}========================================${NC}"

    echo -e "\nPods:"
    kubectl get pods --namespace="${NAMESPACE}" -o wide

    echo -e "\nServices:"
    kubectl get svc --namespace="${NAMESPACE}"

    echo -e "\nHPAs:"
    kubectl get hpa --namespace="${NAMESPACE}"

    echo -e "\n${GREEN}Deployment completed successfully!${NC}"
    echo -e "Access the API Gateway at: $(kubectl get svc api-gateway --namespace="${NAMESPACE}" -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):50051"
}

# Main deployment flow
main() {
    check_prerequisites
    validate_manifests
    deploy_base_resources
    deploy_services
    deploy_autoscaling
    run_health_checks
    show_summary
}

# Run main function
main