ddfa57af11
- Added scaleway_vpc_public_gateway + DHCP + gateway_network to TF (was manually created, now codified with push_default_route=true) - Added scripts/safe-node-replace.sh — one-at-a-time with DNS verification - Added dns-bootstrap-policy.yaml — incident documentation + recovery procedure - Bastion enabled (port 61000) for emergency SSH access Prevention: NEVER replace all nodes at once. Use safe-node-replace.sh. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
79 lines
2.3 KiB
Bash
Executable File
79 lines
2.3 KiB
Bash
Executable File
#!/bin/bash
|
|
# Safe node replacement — one at a time, waits for CoreDNS recovery
|
|
#
|
|
# Usage: ./scripts/safe-node-replace.sh <node-id-1> [node-id-2] [node-id-3]
|
|
#
|
|
# This script prevents the DNS deadlock that occurred on 2026-03-18 by:
|
|
# 1. Replacing ONE node at a time
|
|
# 2. Waiting for CoreDNS to be Running after each replacement
|
|
# 3. Verifying DNS works before proceeding to the next node
|
|
|
|
set -euo pipefail
|
|
|
|
CLUSTER_ID="34a1e3c4-ac35-48c8-ab49-5f6ec4df32c1"
|
|
NAMESPACE="foxhunt"
|
|
|
|
if [ $# -eq 0 ]; then
|
|
echo "Usage: $0 <node-id-1> [node-id-2] [node-id-3]"
|
|
echo ""
|
|
echo "List nodes: scw k8s node list cluster-id=$CLUSTER_ID"
|
|
exit 1
|
|
fi
|
|
|
|
wait_for_coredns() {
|
|
echo " Waiting for CoreDNS to be Running..."
|
|
for i in $(seq 1 60); do
|
|
READY=$(kubectl -n kube-system get pods -l k8s-app=kube-dns --no-headers 2>/dev/null | grep "Running" | wc -l)
|
|
if [ "$READY" -ge 1 ]; then
|
|
echo " CoreDNS: $READY pods Running"
|
|
return 0
|
|
fi
|
|
sleep 10
|
|
done
|
|
echo " ERROR: CoreDNS not Running after 10 minutes!"
|
|
return 1
|
|
}
|
|
|
|
wait_for_node() {
|
|
echo " Waiting for new node to be Ready..."
|
|
for i in $(seq 1 30); do
|
|
TOTAL=$(kubectl get nodes --no-headers 2>/dev/null | grep "Ready" | grep -v "SchedulingDisabled" | wc -l)
|
|
if [ "$TOTAL" -ge "$1" ]; then
|
|
echo " $TOTAL nodes Ready"
|
|
return 0
|
|
fi
|
|
sleep 10
|
|
done
|
|
echo " WARNING: Expected $1 Ready nodes, timeout"
|
|
return 1
|
|
}
|
|
|
|
EXPECTED_NODES=$(kubectl get nodes --no-headers 2>/dev/null | grep "Ready" | wc -l)
|
|
|
|
for NODE_ID in "$@"; do
|
|
echo "=== Replacing node $NODE_ID ==="
|
|
scw k8s node replace "$NODE_ID" || { echo "Replace failed for $NODE_ID"; exit 1; }
|
|
|
|
echo " Waiting for old node to drain..."
|
|
sleep 30
|
|
|
|
wait_for_node "$EXPECTED_NODES"
|
|
wait_for_coredns
|
|
|
|
echo " Verifying DNS..."
|
|
kubectl run dns-verify-$RANDOM --rm -it --restart=Never --image=busybox:1.36 \
|
|
-- sh -c "nslookup google.com > /dev/null 2>&1 && echo DNS_OK || echo DNS_FAIL" 2>/dev/null | grep -q "DNS_OK"
|
|
|
|
if [ $? -eq 0 ]; then
|
|
echo " DNS verified OK"
|
|
else
|
|
echo " WARNING: DNS verification failed — stopping. Fix manually before continuing."
|
|
exit 1
|
|
fi
|
|
|
|
echo " Node $NODE_ID replaced successfully"
|
|
echo ""
|
|
done
|
|
|
|
echo "=== All nodes replaced safely ==="
|