#!/bin/bash
# Safe node replacement — one at a time, waits for CoreDNS recovery
#
# Usage: ./scripts/safe-node-replace.sh <node-id-1> [node-id-2] [node-id-3]
#
# This script prevents the DNS deadlock that occurred on 2026-03-18 by:
# 1. Replacing ONE node at a time
# 2. Waiting for CoreDNS to be Running after each replacement
# 3. Verifying DNS works before proceeding to the next node

set -euo pipefail

CLUSTER_ID="34a1e3c4-ac35-48c8-ab49-5f6ec4df32c1"
NAMESPACE="foxhunt"

if [ $# -eq 0 ]; then
    echo "Usage: $0 <node-id-1> [node-id-2] [node-id-3]"
    echo ""
    echo "List nodes: scw k8s node list cluster-id=$CLUSTER_ID"
    exit 1
fi

wait_for_coredns() {
    echo "  Waiting for CoreDNS to be Running..."
    for i in $(seq 1 60); do
        READY=$(kubectl -n kube-system get pods -l k8s-app=kube-dns --no-headers 2>/dev/null | grep "Running" | wc -l)
        if [ "$READY" -ge 1 ]; then
            echo "  CoreDNS: $READY pods Running"
            return 0
        fi
        sleep 10
    done
    echo "  ERROR: CoreDNS not Running after 10 minutes!"
    return 1
}

wait_for_node() {
    echo "  Waiting for new node to be Ready..."
    for i in $(seq 1 30); do
        TOTAL=$(kubectl get nodes --no-headers 2>/dev/null | grep "Ready" | grep -v "SchedulingDisabled" | wc -l)
        if [ "$TOTAL" -ge "$1" ]; then
            echo "  $TOTAL nodes Ready"
            return 0
        fi
        sleep 10
    done
    echo "  WARNING: Expected $1 Ready nodes, timeout"
    return 1
}

EXPECTED_NODES=$(kubectl get nodes --no-headers 2>/dev/null | grep "Ready" | wc -l)

for NODE_ID in "$@"; do
    echo "=== Replacing node $NODE_ID ==="
    scw k8s node replace "$NODE_ID" || { echo "Replace failed for $NODE_ID"; exit 1; }

    echo "  Waiting for old node to drain..."
    sleep 30

    wait_for_node "$EXPECTED_NODES"
    wait_for_coredns

    echo "  Verifying DNS..."
    kubectl run dns-verify-$RANDOM --rm -it --restart=Never --image=busybox:1.36 \
        -- sh -c "nslookup google.com > /dev/null 2>&1 && echo DNS_OK || echo DNS_FAIL" 2>/dev/null | grep -q "DNS_OK"

    if [ $? -eq 0 ]; then
        echo "  DNS verified OK"
    else
        echo "  WARNING: DNS verification failed — stopping. Fix manually before continuing."
        exit 1
    fi

    echo "  Node $NODE_ID replaced successfully"
    echo ""
done

echo "=== All nodes replaced safely ==="