foxhunt/scripts/deployment/smoke-test.sh

#!/bin/bash
# Foxhunt Post-Deployment Smoke Tests
# Wave 5 W5-4: Comprehensive smoke tests for production deployment
#
# Usage: ./scripts/deployment/smoke-test.sh [namespace]

set -euo pipefail

# Color output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Configuration
NAMESPACE="${1:-foxhunt}"
TIMEOUT=30
TOTAL_TESTS=0
PASSED_TESTS=0
FAILED_TESTS=0

echo -e "${GREEN}========================================${NC}"
echo -e "${GREEN}Foxhunt Smoke Tests${NC}"
echo -e "${GREEN}Namespace: ${NAMESPACE}${NC}"
echo -e "${GREEN}========================================${NC}"

# Function: Run a test
run_test() {
    local test_name=$1
    local test_command=$2

    ((TOTAL_TESTS++))
    echo -e "\n${YELLOW}[Test $TOTAL_TESTS] ${test_name}${NC}"

    if eval "$test_command"; then
        echo -e "${GREEN}✓ PASSED${NC}"
        ((PASSED_TESTS++))
        return 0
    else
        echo -e "${RED}✗ FAILED${NC}"
        ((FAILED_TESTS++))
        return 1
    fi
}

# Test 1: All pods are running
test_pods_running() {
    echo "Checking pod status..."
    kubectl get pods --namespace="${NAMESPACE}" -o json | \
        jq -e '.items | all(.status.phase == "Running")' > /dev/null
}

# Test 2: All deployments are ready
test_deployments_ready() {
    echo "Checking deployment readiness..."
    local deployments=(
        "api-gateway"
        "trading-service"
        "backtesting-service"
        "ml-training-service"
        "trading-agent-service"
    )

    for deployment in "${deployments[@]}"; do
        READY=$(kubectl get deployment "$deployment" --namespace="${NAMESPACE}" \
            -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")
        DESIRED=$(kubectl get deployment "$deployment" --namespace="${NAMESPACE}" \
            -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "1")

        if [ "$READY" != "$DESIRED" ]; then
            echo "Deployment $deployment: $READY/$DESIRED pods ready"
            return 1
        fi
    done

    return 0
}

# Test 3: All services have endpoints
test_services_have_endpoints() {
    echo "Checking service endpoints..."
    local services=(
        "api-gateway"
        "trading-service"
        "backtesting-service"
        "ml-training-service"
        "trading-agent-service"
    )

    for service in "${services[@]}"; do
        ENDPOINTS=$(kubectl get endpoints "$service" --namespace="${NAMESPACE}" \
            -o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null || echo "")

        if [ -z "$ENDPOINTS" ]; then
            echo "Service $service has no endpoints"
            return 1
        fi
    done

    return 0
}

# Test 4: Health checks pass
test_health_checks() {
    echo "Testing health endpoints..."
    local services=(
        "api-gateway:8080:/health/liveness"
        "trading-service:8081:/health"
        "backtesting-service:8082:/health"
        "ml-training-service:8095:/health"
        "trading-agent-service:8083:/health"
    )

    for service_spec in "${services[@]}"; do
        IFS=':' read -r service port path <<< "$service_spec"

        POD=$(kubectl get pods -l app="${service}" \
            --namespace="${NAMESPACE}" \
            -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")

        if [ -z "$POD" ]; then
            echo "No pods found for $service"
            return 1
        fi

        if ! kubectl exec "$POD" --namespace="${NAMESPACE}" -- \
            curl -f -s "http://localhost:${port}${path}" > /dev/null 2>&1; then
            echo "Health check failed for $service"
            return 1
        fi
    done

    return 0
}

# Test 5: Metrics endpoints are accessible
test_metrics_endpoints() {
    echo "Testing metrics endpoints..."
    local services=(
        "api-gateway:9091"
        "trading-service:9092"
        "backtesting-service:9093"
        "ml-training-service:9094"
        "trading-agent-service:9095"
    )

    for service_spec in "${services[@]}"; do
        IFS=':' read -r service port <<< "$service_spec"

        POD=$(kubectl get pods -l app="${service}" \
            --namespace="${NAMESPACE}" \
            -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")

        if [ -z "$POD" ]; then
            echo "No pods found for $service"
            return 1
        fi

        if ! kubectl exec "$POD" --namespace="${NAMESPACE}" -- \
            curl -f -s "http://localhost:${port}/metrics" | grep -q "^#"; then
            echo "Metrics endpoint failed for $service"
            return 1
        fi
    done

    return 0
}

# Test 6: Database connectivity
test_database_connectivity() {
    echo "Testing database connectivity..."

    # Get a pod that can connect to postgres
    POD=$(kubectl get pods -l app="trading-service" \
        --namespace="${NAMESPACE}" \
        -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")

    if [ -z "$POD" ]; then
        echo "No pods found for database test"
        return 1
    fi

    # This is a simplified test - in production, you'd check actual DB connection
    # For now, we just verify the environment variable is set
    kubectl exec "$POD" --namespace="${NAMESPACE}" -- \
        printenv DATABASE_URL > /dev/null 2>&1
}

# Test 7: Redis connectivity
test_redis_connectivity() {
    echo "Testing Redis connectivity..."

    POD=$(kubectl get pods -l app="trading-service" \
        --namespace="${NAMESPACE}" \
        -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")

    if [ -z "$POD" ]; then
        echo "No pods found for Redis test"
        return 1
    fi

    kubectl exec "$POD" --namespace="${NAMESPACE}" -- \
        printenv REDIS_URL > /dev/null 2>&1
}

# Test 8: PVCs are bound
test_pvcs_bound() {
    echo "Testing PVC bindings..."
    local pvcs=(
        "postgres-pvc"
        "redis-pvc"
        "prometheus-pvc"
        "ml-models-pvc"
        "ml-checkpoints-pvc"
        "backtesting-data-pvc"
        "backtesting-results-pvc"
    )

    for pvc in "${pvcs[@]}"; do
        STATUS=$(kubectl get pvc "$pvc" --namespace="${NAMESPACE}" \
            -o jsonpath='{.status.phase}' 2>/dev/null || echo "NotFound")

        if [ "$STATUS" != "Bound" ]; then
            echo "PVC $pvc is not bound (status: $STATUS)"
            return 1
        fi
    done

    return 0
}

# Test 9: HPAs are active
test_hpas_active() {
    echo "Testing HPA status..."
    local hpas=(
        "api-gateway-hpa"
        "trading-service-hpa"
        "ml-training-service-hpa"
    )

    for hpa in "${hpas[@]}"; do
        if ! kubectl get hpa "$hpa" --namespace="${NAMESPACE}" > /dev/null 2>&1; then
            echo "HPA $hpa not found"
            return 1
        fi
    done

    return 0
}

# Test 10: No pods in CrashLoopBackOff
test_no_crashloop() {
    echo "Checking for crash loops..."
    CRASHLOOP=$(kubectl get pods --namespace="${NAMESPACE}" \
        -o json | jq -r '.items[] | select(.status.containerStatuses[]?.state.waiting.reason == "CrashLoopBackOff") | .metadata.name' || echo "")

    if [ -n "$CRASHLOOP" ]; then
        echo "Pods in CrashLoopBackOff: $CRASHLOOP"
        return 1
    fi

    return 0
}

# Function: Show test summary
show_summary() {
    echo -e "\n${GREEN}========================================${NC}"
    echo -e "${GREEN}Smoke Test Summary${NC}"
    echo -e "${GREEN}========================================${NC}"
    echo -e "Total tests: $TOTAL_TESTS"
    echo -e "${GREEN}Passed: $PASSED_TESTS${NC}"

    if [ $FAILED_TESTS -gt 0 ]; then
        echo -e "${RED}Failed: $FAILED_TESTS${NC}"
        echo -e "\n${RED}Smoke tests FAILED${NC}"
        return 1
    else
        echo -e "${RED}Failed: $FAILED_TESTS${NC}"
        echo -e "\n${GREEN}All smoke tests PASSED!${NC}"
        return 0
    fi
}

# Main test flow
main() {
    # Check prerequisites
    if ! command -v kubectl &> /dev/null; then
        echo -e "${RED}Error: kubectl not found${NC}"
        exit 1
    fi

    if ! command -v jq &> /dev/null; then
        echo -e "${RED}Error: jq not found${NC}"
        exit 1
    fi

    # Run all tests
    run_test "All pods are running" "test_pods_running" || true
    run_test "All deployments are ready" "test_deployments_ready" || true
    run_test "All services have endpoints" "test_services_have_endpoints" || true
    run_test "Health checks pass" "test_health_checks" || true
    run_test "Metrics endpoints accessible" "test_metrics_endpoints" || true
    run_test "Database connectivity" "test_database_connectivity" || true
    run_test "Redis connectivity" "test_redis_connectivity" || true
    run_test "PVCs are bound" "test_pvcs_bound" || true
    run_test "HPAs are active" "test_hpas_active" || true
    run_test "No pods in CrashLoopBackOff" "test_no_crashloop" || true

    # Show summary and exit with appropriate code
    if show_summary; then
        exit 0
    else
        exit 1
    fi
}

# Run main function
main