#!/usr/bin/env bash # Foxhunt Broker Gateway Service - Production Deployment Script # Builds multi-arch Docker image, pushes to registry, deploys to Kubernetes # Includes smoke tests and automatic rollback on failure set -euo pipefail # ============================================================================ # Configuration # ============================================================================ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" SERVICE_DIR="${PROJECT_ROOT}/services/broker_gateway_service" K8S_DIR="${SERVICE_DIR}/k8s" # Docker configuration DOCKER_REGISTRY="${DOCKER_REGISTRY:-jgrusewski}" IMAGE_NAME="${IMAGE_NAME:-foxhunt-broker-gateway}" IMAGE_TAG="${IMAGE_TAG:-latest}" FULL_IMAGE_NAME="${DOCKER_REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}" # Kubernetes configuration NAMESPACE="${NAMESPACE:-foxhunt}" DEPLOYMENT_NAME="broker-gateway-service" TIMEOUT="${TIMEOUT:-300}" # 5 minutes # Build configuration BUILD_PLATFORMS="${BUILD_PLATFORMS:-linux/amd64,linux/arm64}" SKIP_BUILD="${SKIP_BUILD:-false}" SKIP_PUSH="${SKIP_PUSH:-false}" SKIP_TESTS="${SKIP_TESTS:-false}" DRY_RUN="${DRY_RUN:-false}" # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color # ============================================================================ # Helper Functions # ============================================================================ log_info() { echo -e "${BLUE}[INFO]${NC} $*" } log_success() { echo -e "${GREEN}[SUCCESS]${NC} $*" } log_warn() { echo -e "${YELLOW}[WARN]${NC} $*" } log_error() { echo -e "${RED}[ERROR]${NC} $*" } # Check if command exists command_exists() { command -v "$1" >/dev/null 2>&1 } # Check prerequisites check_prerequisites() { log_info "Checking prerequisites..." local missing_tools=() if ! command_exists docker; then missing_tools+=("docker") fi if ! command_exists kubectl; then missing_tools+=("kubectl") fi if [[ "$SKIP_BUILD" == "false" ]]; then if ! docker buildx version >/dev/null 2>&1; then log_error "Docker BuildKit is not available. Install with: docker buildx create --use" exit 1 fi fi if [[ ${#missing_tools[@]} -gt 0 ]]; then log_error "Missing required tools: ${missing_tools[*]}" log_error "Please install missing tools and try again" exit 1 fi log_success "All prerequisites are installed" } # Get git commit hash get_git_commit() { if command_exists git && git rev-parse --git-dir >/dev/null 2>&1; then git rev-parse --short HEAD else echo "unknown" fi } # Get build date get_build_date() { date -u +"%Y-%m-%dT%H:%M:%SZ" } # Build Docker image build_docker_image() { if [[ "$SKIP_BUILD" == "true" ]]; then log_warn "Skipping Docker build (SKIP_BUILD=true)" return 0 fi log_info "Building Docker image: ${FULL_IMAGE_NAME}" log_info "Platforms: ${BUILD_PLATFORMS}" local git_commit git_commit=$(get_git_commit) local build_date build_date=$(get_build_date) cd "${PROJECT_ROOT}" # Build multi-arch image with BuildKit docker buildx build \ --platform "${BUILD_PLATFORMS}" \ --file "${SERVICE_DIR}/Dockerfile.production" \ --tag "${FULL_IMAGE_NAME}" \ --tag "${DOCKER_REGISTRY}/${IMAGE_NAME}:${git_commit}" \ --build-arg "GIT_COMMIT=${git_commit}" \ --build-arg "BUILD_DATE=${build_date}" \ --build-arg "VERSION=${IMAGE_TAG}" \ ${SKIP_PUSH:+--load} \ ${SKIP_PUSH:---push} \ . if [[ "$SKIP_PUSH" == "true" ]]; then log_success "Docker image built locally (not pushed)" else log_success "Docker image built and pushed: ${FULL_IMAGE_NAME}" fi } # Check Docker image size check_image_size() { if [[ "$SKIP_BUILD" == "true" ]]; then return 0 fi log_info "Checking image size..." local image_size image_size=$(docker image inspect "${FULL_IMAGE_NAME}" --format='{{.Size}}' 2>/dev/null || echo "0") if [[ "$image_size" -eq 0 ]]; then log_warn "Could not determine image size (image may not be loaded locally)" return 0 fi local size_mb=$((image_size / 1024 / 1024)) log_info "Image size: ${size_mb}MB" # Warn if image is larger than expected (>100MB) if [[ $size_mb -gt 100 ]]; then log_warn "Image size is larger than expected (${size_mb}MB > 100MB)" log_warn "Consider optimizing the image" else log_success "Image size is optimal (${size_mb}MB)" fi } # Check Kubernetes connectivity check_k8s_connectivity() { log_info "Checking Kubernetes connectivity..." if ! kubectl cluster-info >/dev/null 2>&1; then log_error "Cannot connect to Kubernetes cluster" log_error "Check your kubeconfig and cluster status" exit 1 fi log_success "Connected to Kubernetes cluster" # Check if namespace exists, create if not if ! kubectl get namespace "${NAMESPACE}" >/dev/null 2>&1; then log_warn "Namespace '${NAMESPACE}' does not exist" if [[ "$DRY_RUN" == "false" ]]; then log_info "Creating namespace '${NAMESPACE}'..." kubectl create namespace "${NAMESPACE}" log_success "Namespace created" else log_info "[DRY-RUN] Would create namespace '${NAMESPACE}'" fi fi } # Apply Kubernetes manifests apply_k8s_manifests() { log_info "Applying Kubernetes manifests..." local manifests=( "configmap.yaml" "secret.yaml" "service.yaml" "deployment.yaml" "hpa.yaml" ) for manifest in "${manifests[@]}"; do local manifest_path="${K8S_DIR}/${manifest}" if [[ ! -f "$manifest_path" ]]; then log_error "Manifest not found: ${manifest_path}" exit 1 fi log_info "Applying ${manifest}..." if [[ "$DRY_RUN" == "true" ]]; then kubectl apply -f "${manifest_path}" --namespace="${NAMESPACE}" --dry-run=client log_info "[DRY-RUN] Would apply ${manifest}" else kubectl apply -f "${manifest_path}" --namespace="${NAMESPACE}" log_success "Applied ${manifest}" fi done log_success "All manifests applied" } # Wait for rollout to complete wait_for_rollout() { if [[ "$DRY_RUN" == "true" ]]; then log_info "[DRY-RUN] Would wait for rollout to complete" return 0 fi log_info "Waiting for rollout to complete (timeout: ${TIMEOUT}s)..." if kubectl rollout status statefulset/"${DEPLOYMENT_NAME}" \ --namespace="${NAMESPACE}" \ --timeout="${TIMEOUT}s"; then log_success "Rollout completed successfully" return 0 else log_error "Rollout failed or timed out" return 1 fi } # Get pod status get_pod_status() { kubectl get pods \ --namespace="${NAMESPACE}" \ --selector="app=${DEPLOYMENT_NAME}" \ --output=json | jq -r '.items[] | "\(.metadata.name): \(.status.phase) (Ready: \(.status.conditions[] | select(.type=="Ready") | .status))"' } # Run smoke tests run_smoke_tests() { if [[ "$SKIP_TESTS" == "true" ]]; then log_warn "Skipping smoke tests (SKIP_TESTS=true)" return 0 fi if [[ "$DRY_RUN" == "true" ]]; then log_info "[DRY-RUN] Would run smoke tests" return 0 fi log_info "Running smoke tests..." # Get pod name local pod_name pod_name=$(kubectl get pods \ --namespace="${NAMESPACE}" \ --selector="app=${DEPLOYMENT_NAME}" \ --output=jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") if [[ -z "$pod_name" ]]; then log_error "No pods found for deployment ${DEPLOYMENT_NAME}" return 1 fi log_info "Testing pod: ${pod_name}" # Test 1: Health check endpoint log_info "Test 1: Health check endpoint..." if kubectl exec "${pod_name}" --namespace="${NAMESPACE}" -- \ curl -sf http://localhost:8086/health >/dev/null; then log_success "Health check passed" else log_error "Health check failed" return 1 fi # Test 2: gRPC health probe log_info "Test 2: gRPC health probe..." if kubectl exec "${pod_name}" --namespace="${NAMESPACE}" -- \ /usr/local/bin/grpc_health_probe -addr=localhost:50056; then log_success "gRPC health probe passed" else log_error "gRPC health probe failed" return 1 fi # Test 3: Metrics endpoint log_info "Test 3: Metrics endpoint..." if kubectl exec "${pod_name}" --namespace="${NAMESPACE}" -- \ curl -sf http://localhost:9096/metrics >/dev/null; then log_success "Metrics endpoint passed" else log_warn "Metrics endpoint failed (non-critical)" fi log_success "All smoke tests passed" return 0 } # Rollback deployment rollback_deployment() { log_error "Deployment failed. Rolling back..." if [[ "$DRY_RUN" == "true" ]]; then log_info "[DRY-RUN] Would rollback deployment" return 0 fi kubectl rollout undo statefulset/"${DEPLOYMENT_NAME}" --namespace="${NAMESPACE}" log_info "Waiting for rollback to complete..." kubectl rollout status statefulset/"${DEPLOYMENT_NAME}" \ --namespace="${NAMESPACE}" \ --timeout="${TIMEOUT}s" log_warn "Rollback completed" } # Print deployment summary print_summary() { log_info "===================================================================" log_info "Deployment Summary" log_info "===================================================================" log_info "Image: ${FULL_IMAGE_NAME}" log_info "Namespace: ${NAMESPACE}" log_info "Deployment: ${DEPLOYMENT_NAME}" log_info "-------------------------------------------------------------------" log_info "Pod Status:" get_pod_status log_info "===================================================================" } # ============================================================================ # Main Deployment Flow # ============================================================================ main() { log_info "Starting Broker Gateway Service deployment..." log_info "Image: ${FULL_IMAGE_NAME}" log_info "Namespace: ${NAMESPACE}" log_info "Dry-run: ${DRY_RUN}" # Step 1: Check prerequisites check_prerequisites # Step 2: Build Docker image build_docker_image # Step 3: Check image size check_image_size # Step 4: Check Kubernetes connectivity check_k8s_connectivity # Step 5: Apply Kubernetes manifests apply_k8s_manifests # Step 6: Wait for rollout to complete if ! wait_for_rollout; then rollback_deployment exit 1 fi # Step 7: Run smoke tests if ! run_smoke_tests; then log_error "Smoke tests failed" rollback_deployment exit 1 fi # Step 8: Print deployment summary print_summary log_success "Broker Gateway Service deployment completed successfully!" } # ============================================================================ # Script Entry Point # ============================================================================ # Parse command-line arguments while [[ $# -gt 0 ]]; do case $1 in --skip-build) SKIP_BUILD=true shift ;; --skip-push) SKIP_PUSH=true shift ;; --skip-tests) SKIP_TESTS=true shift ;; --dry-run) DRY_RUN=true shift ;; --namespace) NAMESPACE="$2" shift 2 ;; --image-tag) IMAGE_TAG="$2" shift 2 ;; --timeout) TIMEOUT="$2" shift 2 ;; --help) echo "Usage: $0 [options]" echo "" echo "Options:" echo " --skip-build Skip Docker image build" echo " --skip-push Build locally but do not push to registry" echo " --skip-tests Skip smoke tests" echo " --dry-run Show what would be done without making changes" echo " --namespace NAME Kubernetes namespace (default: foxhunt)" echo " --image-tag TAG Docker image tag (default: latest)" echo " --timeout SEC Rollout timeout in seconds (default: 300)" echo " --help Show this help message" exit 0 ;; *) log_error "Unknown option: $1" log_error "Use --help for usage information" exit 1 ;; esac done # Run main deployment flow main