diff --git a/infra/k8s/argo/dns-bootstrap-policy.yaml b/infra/k8s/argo/dns-bootstrap-policy.yaml new file mode 100644 index 000000000..ef73cc12b --- /dev/null +++ b/infra/k8s/argo/dns-bootstrap-policy.yaml @@ -0,0 +1,25 @@ +# DNS Bootstrap Prevention — Prevent CoreDNS chicken-and-egg deadlock +# +# ROOT CAUSE (2026-03-18 incident): +# 1. Rolling replacement of ALL platform nodes simultaneously +# 2. New nodes boot with /etc/resolv.conf → 10.32.0.10 (kube-dns only) +# 3. CoreDNS pods need to pull images → need DNS → need CoreDNS +# 4. Cilium BPF socket LB returns EPERM when kube-dns has no endpoints +# 5. Complete cluster DNS deadlock — nothing can pull images +# +# PREVENTION: +# 1. VPC DefaultRoutePropagation MUST be enabled (API, not Terraform yet) +# 2. NEVER replace ALL nodes at once — one at a time, wait for CoreDNS +# 3. Do NOT create CiliumClusterwideNetworkPolicy with egress rules +# (enables default deny, makes the problem worse) +# 4. bpf-lb-sock MUST remain "true" — disabling breaks ALL service routing +# 5. Keep bastion enabled on VPC gateway for emergency SSH access +# +# RECOVERY PROCEDURE: +# 1. SSH via bastion: ssh -o ProxyCommand="ssh -W %h:%p -p 61000 bastion@" root@ +# 2. On the node: /usr/local/bin/crictl pull +# 3. kubectl delete pods -l k8s-app=kube-dns --force +# 4. CoreDNS starts from cached image → DNS recovers → cascade +# +# DATA PROTECTION: +# All PVs set to Retain reclaim policy — data survives cluster deletion diff --git a/infra/modules/kapsule/main.tf b/infra/modules/kapsule/main.tf index 510afbcfd..9e30c54fb 100644 --- a/infra/modules/kapsule/main.tf +++ b/infra/modules/kapsule/main.tf @@ -3,6 +3,32 @@ resource "scaleway_vpc_private_network" "foxhunt" { region = var.region } +# VPC Public Gateway — provides NAT (masquerade) for private nodes +# and DHCP with default route propagation to prevent DNS deadlock. +# See incident_dns_deadlock.md for why this is critical. +resource "scaleway_vpc_public_gateway" "foxhunt" { + name = "${var.cluster_name}-gw" + type = "VPC-GW-S" + zone = "${var.region}-2" + bastion_enabled = true + bastion_port = 61000 +} + +resource "scaleway_vpc_public_gateway_dhcp" "foxhunt" { + subnet = "172.16.0.0/22" + push_default_route = true + push_dns_server = true + zone = "${var.region}-2" +} + +resource "scaleway_vpc_gateway_network" "foxhunt" { + gateway_id = scaleway_vpc_public_gateway.foxhunt.id + private_network_id = scaleway_vpc_private_network.foxhunt.id + dhcp_id = scaleway_vpc_public_gateway_dhcp.foxhunt.id + enable_masquerade = true + zone = "${var.region}-2" +} + resource "scaleway_k8s_cluster" "foxhunt" { name = var.cluster_name version = var.k8s_version diff --git a/scripts/safe-node-replace.sh b/scripts/safe-node-replace.sh new file mode 100755 index 000000000..2a25fd4cb --- /dev/null +++ b/scripts/safe-node-replace.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Safe node replacement — one at a time, waits for CoreDNS recovery +# +# Usage: ./scripts/safe-node-replace.sh [node-id-2] [node-id-3] +# +# This script prevents the DNS deadlock that occurred on 2026-03-18 by: +# 1. Replacing ONE node at a time +# 2. Waiting for CoreDNS to be Running after each replacement +# 3. Verifying DNS works before proceeding to the next node + +set -euo pipefail + +CLUSTER_ID="34a1e3c4-ac35-48c8-ab49-5f6ec4df32c1" +NAMESPACE="foxhunt" + +if [ $# -eq 0 ]; then + echo "Usage: $0 [node-id-2] [node-id-3]" + echo "" + echo "List nodes: scw k8s node list cluster-id=$CLUSTER_ID" + exit 1 +fi + +wait_for_coredns() { + echo " Waiting for CoreDNS to be Running..." + for i in $(seq 1 60); do + READY=$(kubectl -n kube-system get pods -l k8s-app=kube-dns --no-headers 2>/dev/null | grep "Running" | wc -l) + if [ "$READY" -ge 1 ]; then + echo " CoreDNS: $READY pods Running" + return 0 + fi + sleep 10 + done + echo " ERROR: CoreDNS not Running after 10 minutes!" + return 1 +} + +wait_for_node() { + echo " Waiting for new node to be Ready..." + for i in $(seq 1 30); do + TOTAL=$(kubectl get nodes --no-headers 2>/dev/null | grep "Ready" | grep -v "SchedulingDisabled" | wc -l) + if [ "$TOTAL" -ge "$1" ]; then + echo " $TOTAL nodes Ready" + return 0 + fi + sleep 10 + done + echo " WARNING: Expected $1 Ready nodes, timeout" + return 1 +} + +EXPECTED_NODES=$(kubectl get nodes --no-headers 2>/dev/null | grep "Ready" | wc -l) + +for NODE_ID in "$@"; do + echo "=== Replacing node $NODE_ID ===" + scw k8s node replace "$NODE_ID" || { echo "Replace failed for $NODE_ID"; exit 1; } + + echo " Waiting for old node to drain..." + sleep 30 + + wait_for_node "$EXPECTED_NODES" + wait_for_coredns + + echo " Verifying DNS..." + kubectl run dns-verify-$RANDOM --rm -it --restart=Never --image=busybox:1.36 \ + -- sh -c "nslookup google.com > /dev/null 2>&1 && echo DNS_OK || echo DNS_FAIL" 2>/dev/null | grep -q "DNS_OK" + + if [ $? -eq 0 ]; then + echo " DNS verified OK" + else + echo " WARNING: DNS verification failed — stopping. Fix manually before continuing." + exit 1 + fi + + echo " Node $NODE_ID replaced successfully" + echo "" +done + +echo "=== All nodes replaced safely ==="