foxhunt/scripts/deploy_broker_gateway.sh

#!/usr/bin/env bash
# Foxhunt Broker Gateway Service - Production Deployment Script
# Builds multi-arch Docker image, pushes to registry, deploys to Kubernetes
# Includes smoke tests and automatic rollback on failure

set -euo pipefail

# ============================================================================
# Configuration
# ============================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
SERVICE_DIR="${PROJECT_ROOT}/services/broker_gateway_service"
K8S_DIR="${SERVICE_DIR}/k8s"

# Docker configuration
DOCKER_REGISTRY="${DOCKER_REGISTRY:-jgrusewski}"
IMAGE_NAME="${IMAGE_NAME:-foxhunt-broker-gateway}"
IMAGE_TAG="${IMAGE_TAG:-latest}"
FULL_IMAGE_NAME="${DOCKER_REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}"

# Kubernetes configuration
NAMESPACE="${NAMESPACE:-foxhunt}"
DEPLOYMENT_NAME="broker-gateway-service"
TIMEOUT="${TIMEOUT:-300}"  # 5 minutes

# Build configuration
BUILD_PLATFORMS="${BUILD_PLATFORMS:-linux/amd64,linux/arm64}"
SKIP_BUILD="${SKIP_BUILD:-false}"
SKIP_PUSH="${SKIP_PUSH:-false}"
SKIP_TESTS="${SKIP_TESTS:-false}"
DRY_RUN="${DRY_RUN:-false}"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# ============================================================================
# Helper Functions
# ============================================================================

log_info() {
    echo -e "${BLUE}[INFO]${NC} $*"
}

log_success() {
    echo -e "${GREEN}[SUCCESS]${NC} $*"
}

log_warn() {
    echo -e "${YELLOW}[WARN]${NC} $*"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $*"
}

# Check if command exists
command_exists() {
    command -v "$1" >/dev/null 2>&1
}

# Check prerequisites
check_prerequisites() {
    log_info "Checking prerequisites..."

    local missing_tools=()

    if ! command_exists docker; then
        missing_tools+=("docker")
    fi

    if ! command_exists kubectl; then
        missing_tools+=("kubectl")
    fi

    if [[ "$SKIP_BUILD" == "false" ]]; then
        if ! docker buildx version >/dev/null 2>&1; then
            log_error "Docker BuildKit is not available. Install with: docker buildx create --use"
            exit 1
        fi
    fi

    if [[ ${#missing_tools[@]} -gt 0 ]]; then
        log_error "Missing required tools: ${missing_tools[*]}"
        log_error "Please install missing tools and try again"
        exit 1
    fi

    log_success "All prerequisites are installed"
}

# Get git commit hash
get_git_commit() {
    if command_exists git && git rev-parse --git-dir >/dev/null 2>&1; then
        git rev-parse --short HEAD
    else
        echo "unknown"
    fi
}

# Get build date
get_build_date() {
    date -u +"%Y-%m-%dT%H:%M:%SZ"
}

# Build Docker image
build_docker_image() {
    if [[ "$SKIP_BUILD" == "true" ]]; then
        log_warn "Skipping Docker build (SKIP_BUILD=true)"
        return 0
    fi

    log_info "Building Docker image: ${FULL_IMAGE_NAME}"
    log_info "Platforms: ${BUILD_PLATFORMS}"

    local git_commit
    git_commit=$(get_git_commit)
    local build_date
    build_date=$(get_build_date)

    cd "${PROJECT_ROOT}"

    # Build multi-arch image with BuildKit
    docker buildx build \
        --platform "${BUILD_PLATFORMS}" \
        --file "${SERVICE_DIR}/Dockerfile.production" \
        --tag "${FULL_IMAGE_NAME}" \
        --tag "${DOCKER_REGISTRY}/${IMAGE_NAME}:${git_commit}" \
        --build-arg "GIT_COMMIT=${git_commit}" \
        --build-arg "BUILD_DATE=${build_date}" \
        --build-arg "VERSION=${IMAGE_TAG}" \
        ${SKIP_PUSH:+--load} \
        ${SKIP_PUSH:---push} \
        .

    if [[ "$SKIP_PUSH" == "true" ]]; then
        log_success "Docker image built locally (not pushed)"
    else
        log_success "Docker image built and pushed: ${FULL_IMAGE_NAME}"
    fi
}

# Check Docker image size
check_image_size() {
    if [[ "$SKIP_BUILD" == "true" ]]; then
        return 0
    fi

    log_info "Checking image size..."

    local image_size
    image_size=$(docker image inspect "${FULL_IMAGE_NAME}" --format='{{.Size}}' 2>/dev/null || echo "0")

    if [[ "$image_size" -eq 0 ]]; then
        log_warn "Could not determine image size (image may not be loaded locally)"
        return 0
    fi

    local size_mb=$((image_size / 1024 / 1024))
    log_info "Image size: ${size_mb}MB"

    # Warn if image is larger than expected (>100MB)
    if [[ $size_mb -gt 100 ]]; then
        log_warn "Image size is larger than expected (${size_mb}MB > 100MB)"
        log_warn "Consider optimizing the image"
    else
        log_success "Image size is optimal (${size_mb}MB)"
    fi
}

# Check Kubernetes connectivity
check_k8s_connectivity() {
    log_info "Checking Kubernetes connectivity..."

    if ! kubectl cluster-info >/dev/null 2>&1; then
        log_error "Cannot connect to Kubernetes cluster"
        log_error "Check your kubeconfig and cluster status"
        exit 1
    fi

    log_success "Connected to Kubernetes cluster"

    # Check if namespace exists, create if not
    if ! kubectl get namespace "${NAMESPACE}" >/dev/null 2>&1; then
        log_warn "Namespace '${NAMESPACE}' does not exist"
        if [[ "$DRY_RUN" == "false" ]]; then
            log_info "Creating namespace '${NAMESPACE}'..."
            kubectl create namespace "${NAMESPACE}"
            log_success "Namespace created"
        else
            log_info "[DRY-RUN] Would create namespace '${NAMESPACE}'"
        fi
    fi
}

# Apply Kubernetes manifests
apply_k8s_manifests() {
    log_info "Applying Kubernetes manifests..."

    local manifests=(
        "configmap.yaml"
        "secret.yaml"
        "service.yaml"
        "deployment.yaml"
        "hpa.yaml"
    )

    for manifest in "${manifests[@]}"; do
        local manifest_path="${K8S_DIR}/${manifest}"
        if [[ ! -f "$manifest_path" ]]; then
            log_error "Manifest not found: ${manifest_path}"
            exit 1
        fi

        log_info "Applying ${manifest}..."
        if [[ "$DRY_RUN" == "true" ]]; then
            kubectl apply -f "${manifest_path}" --namespace="${NAMESPACE}" --dry-run=client
            log_info "[DRY-RUN] Would apply ${manifest}"
        else
            kubectl apply -f "${manifest_path}" --namespace="${NAMESPACE}"
            log_success "Applied ${manifest}"
        fi
    done

    log_success "All manifests applied"
}

# Wait for rollout to complete
wait_for_rollout() {
    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY-RUN] Would wait for rollout to complete"
        return 0
    fi

    log_info "Waiting for rollout to complete (timeout: ${TIMEOUT}s)..."

    if kubectl rollout status statefulset/"${DEPLOYMENT_NAME}" \
        --namespace="${NAMESPACE}" \
        --timeout="${TIMEOUT}s"; then
        log_success "Rollout completed successfully"
        return 0
    else
        log_error "Rollout failed or timed out"
        return 1
    fi
}

# Get pod status
get_pod_status() {
    kubectl get pods \
        --namespace="${NAMESPACE}" \
        --selector="app=${DEPLOYMENT_NAME}" \
        --output=json |
        jq -r '.items[] | "\(.metadata.name): \(.status.phase) (Ready: \(.status.conditions[] | select(.type=="Ready") | .status))"'
}

# Run smoke tests
run_smoke_tests() {
    if [[ "$SKIP_TESTS" == "true" ]]; then
        log_warn "Skipping smoke tests (SKIP_TESTS=true)"
        return 0
    fi

    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY-RUN] Would run smoke tests"
        return 0
    fi

    log_info "Running smoke tests..."

    # Get pod name
    local pod_name
    pod_name=$(kubectl get pods \
        --namespace="${NAMESPACE}" \
        --selector="app=${DEPLOYMENT_NAME}" \
        --output=jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")

    if [[ -z "$pod_name" ]]; then
        log_error "No pods found for deployment ${DEPLOYMENT_NAME}"
        return 1
    fi

    log_info "Testing pod: ${pod_name}"

    # Test 1: Health check endpoint
    log_info "Test 1: Health check endpoint..."
    if kubectl exec "${pod_name}" --namespace="${NAMESPACE}" -- \
        curl -sf http://localhost:8086/health >/dev/null; then
        log_success "Health check passed"
    else
        log_error "Health check failed"
        return 1
    fi

    # Test 2: gRPC health probe
    log_info "Test 2: gRPC health probe..."
    if kubectl exec "${pod_name}" --namespace="${NAMESPACE}" -- \
        /usr/local/bin/grpc_health_probe -addr=localhost:50056; then
        log_success "gRPC health probe passed"
    else
        log_error "gRPC health probe failed"
        return 1
    fi

    # Test 3: Metrics endpoint
    log_info "Test 3: Metrics endpoint..."
    if kubectl exec "${pod_name}" --namespace="${NAMESPACE}" -- \
        curl -sf http://localhost:9096/metrics >/dev/null; then
        log_success "Metrics endpoint passed"
    else
        log_warn "Metrics endpoint failed (non-critical)"
    fi

    log_success "All smoke tests passed"
    return 0
}

# Rollback deployment
rollback_deployment() {
    log_error "Deployment failed. Rolling back..."

    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY-RUN] Would rollback deployment"
        return 0
    fi

    kubectl rollout undo statefulset/"${DEPLOYMENT_NAME}" --namespace="${NAMESPACE}"

    log_info "Waiting for rollback to complete..."
    kubectl rollout status statefulset/"${DEPLOYMENT_NAME}" \
        --namespace="${NAMESPACE}" \
        --timeout="${TIMEOUT}s"

    log_warn "Rollback completed"
}

# Print deployment summary
print_summary() {
    log_info "==================================================================="
    log_info "Deployment Summary"
    log_info "==================================================================="
    log_info "Image:      ${FULL_IMAGE_NAME}"
    log_info "Namespace:  ${NAMESPACE}"
    log_info "Deployment: ${DEPLOYMENT_NAME}"
    log_info "-------------------------------------------------------------------"
    log_info "Pod Status:"
    get_pod_status
    log_info "==================================================================="
}

# ============================================================================
# Main Deployment Flow
# ============================================================================

main() {
    log_info "Starting Broker Gateway Service deployment..."
    log_info "Image: ${FULL_IMAGE_NAME}"
    log_info "Namespace: ${NAMESPACE}"
    log_info "Dry-run: ${DRY_RUN}"

    # Step 1: Check prerequisites
    check_prerequisites

    # Step 2: Build Docker image
    build_docker_image

    # Step 3: Check image size
    check_image_size

    # Step 4: Check Kubernetes connectivity
    check_k8s_connectivity

    # Step 5: Apply Kubernetes manifests
    apply_k8s_manifests

    # Step 6: Wait for rollout to complete
    if ! wait_for_rollout; then
        rollback_deployment
        exit 1
    fi

    # Step 7: Run smoke tests
    if ! run_smoke_tests; then
        log_error "Smoke tests failed"
        rollback_deployment
        exit 1
    fi

    # Step 8: Print deployment summary
    print_summary

    log_success "Broker Gateway Service deployment completed successfully!"
}

# ============================================================================
# Script Entry Point
# ============================================================================

# Parse command-line arguments
while [[ $# -gt 0 ]]; do
    case $1 in
    --skip-build)
        SKIP_BUILD=true
        shift
        ;;
    --skip-push)
        SKIP_PUSH=true
        shift
        ;;
    --skip-tests)
        SKIP_TESTS=true
        shift
        ;;
    --dry-run)
        DRY_RUN=true
        shift
        ;;
    --namespace)
        NAMESPACE="$2"
        shift 2
        ;;
    --image-tag)
        IMAGE_TAG="$2"
        shift 2
        ;;
    --timeout)
        TIMEOUT="$2"
        shift 2
        ;;
    --help)
        echo "Usage: $0 [options]"
        echo ""
        echo "Options:"
        echo "  --skip-build      Skip Docker image build"
        echo "  --skip-push       Build locally but do not push to registry"
        echo "  --skip-tests      Skip smoke tests"
        echo "  --dry-run         Show what would be done without making changes"
        echo "  --namespace NAME  Kubernetes namespace (default: foxhunt)"
        echo "  --image-tag TAG   Docker image tag (default: latest)"
        echo "  --timeout SEC     Rollout timeout in seconds (default: 300)"
        echo "  --help            Show this help message"
        exit 0
        ;;
    *)
        log_error "Unknown option: $1"
        log_error "Use --help for usage information"
        exit 1
        ;;
    esac
done

# Run main deployment flow
main