infra: VPC gateway in Terraform + DNS deadlock prevention

- Added scaleway_vpc_public_gateway + DHCP + gateway_network to TF
  (was manually created, now codified with push_default_route=true)
- Added scripts/safe-node-replace.sh — one-at-a-time with DNS verification
- Added dns-bootstrap-policy.yaml — incident documentation + recovery procedure
- Bastion enabled (port 61000) for emergency SSH access

Prevention: NEVER replace all nodes at once. Use safe-node-replace.sh.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

infra/k8s/argo/dns-bootstrap-policy.yaml

@@ -0,0 +1,25 @@
+# DNS Bootstrap Prevention — Prevent CoreDNS chicken-and-egg deadlock
+#
+# ROOT CAUSE (2026-03-18 incident):
+# 1. Rolling replacement of ALL platform nodes simultaneously
+# 2. New nodes boot with /etc/resolv.conf → 10.32.0.10 (kube-dns only)
+# 3. CoreDNS pods need to pull images → need DNS → need CoreDNS
+# 4. Cilium BPF socket LB returns EPERM when kube-dns has no endpoints
+# 5. Complete cluster DNS deadlock — nothing can pull images
+#
+# PREVENTION:
+# 1. VPC DefaultRoutePropagation MUST be enabled (API, not Terraform yet)
+# 2. NEVER replace ALL nodes at once — one at a time, wait for CoreDNS
+# 3. Do NOT create CiliumClusterwideNetworkPolicy with egress rules
+#    (enables default deny, makes the problem worse)
+# 4. bpf-lb-sock MUST remain "true" — disabling breaks ALL service routing
+# 5. Keep bastion enabled on VPC gateway for emergency SSH access
+#
+# RECOVERY PROCEDURE:
+# 1. SSH via bastion: ssh -o ProxyCommand="ssh -W %h:%p -p 61000 bastion@<gw-public-ip>" root@<node-private-ip>
+# 2. On the node: /usr/local/bin/crictl pull <coredns-image>
+# 3. kubectl delete pods -l k8s-app=kube-dns --force
+# 4. CoreDNS starts from cached image → DNS recovers → cascade
+#
+# DATA PROTECTION:
+# All PVs set to Retain reclaim policy — data survives cluster deletion

infra/modules/kapsule/main.tf

@@ -3,6 +3,32 @@ resource "scaleway_vpc_private_network" "foxhunt" {
   region = var.region
 }
 
+# VPC Public Gateway — provides NAT (masquerade) for private nodes
+# and DHCP with default route propagation to prevent DNS deadlock.
+# See incident_dns_deadlock.md for why this is critical.
+resource "scaleway_vpc_public_gateway" "foxhunt" {
+  name            = "${var.cluster_name}-gw"
+  type            = "VPC-GW-S"
+  zone            = "${var.region}-2"
+  bastion_enabled = true
+  bastion_port    = 61000
+}
+
+resource "scaleway_vpc_public_gateway_dhcp" "foxhunt" {
+  subnet             = "172.16.0.0/22"
+  push_default_route = true
+  push_dns_server    = true
+  zone               = "${var.region}-2"
+}
+
+resource "scaleway_vpc_gateway_network" "foxhunt" {
+  gateway_id         = scaleway_vpc_public_gateway.foxhunt.id
+  private_network_id = scaleway_vpc_private_network.foxhunt.id
+  dhcp_id            = scaleway_vpc_public_gateway_dhcp.foxhunt.id
+  enable_masquerade  = true
+  zone               = "${var.region}-2"
+}
+
 resource "scaleway_k8s_cluster" "foxhunt" {
   name                        = var.cluster_name
   version                     = var.k8s_version

scripts/safe-node-replace.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+# Safe node replacement — one at a time, waits for CoreDNS recovery
+#
+# Usage: ./scripts/safe-node-replace.sh <node-id-1> [node-id-2] [node-id-3]
+#
+# This script prevents the DNS deadlock that occurred on 2026-03-18 by:
+# 1. Replacing ONE node at a time
+# 2. Waiting for CoreDNS to be Running after each replacement
+# 3. Verifying DNS works before proceeding to the next node
+
+set -euo pipefail
+
+CLUSTER_ID="34a1e3c4-ac35-48c8-ab49-5f6ec4df32c1"
+NAMESPACE="foxhunt"
+
+if [ $# -eq 0 ]; then
+    echo "Usage: $0 <node-id-1> [node-id-2] [node-id-3]"
+    echo ""
+    echo "List nodes: scw k8s node list cluster-id=$CLUSTER_ID"
+    exit 1
+fi
+
+wait_for_coredns() {
+    echo "  Waiting for CoreDNS to be Running..."
+    for i in $(seq 1 60); do
+        READY=$(kubectl -n kube-system get pods -l k8s-app=kube-dns --no-headers 2>/dev/null | grep "Running" | wc -l)
+        if [ "$READY" -ge 1 ]; then
+            echo "  CoreDNS: $READY pods Running"
+            return 0
+        fi
+        sleep 10
+    done
+    echo "  ERROR: CoreDNS not Running after 10 minutes!"
+    return 1
+}
+
+wait_for_node() {
+    echo "  Waiting for new node to be Ready..."
+    for i in $(seq 1 30); do
+        TOTAL=$(kubectl get nodes --no-headers 2>/dev/null | grep "Ready" | grep -v "SchedulingDisabled" | wc -l)
+        if [ "$TOTAL" -ge "$1" ]; then
+            echo "  $TOTAL nodes Ready"
+            return 0
+        fi
+        sleep 10
+    done
+    echo "  WARNING: Expected $1 Ready nodes, timeout"
+    return 1
+}
+
+EXPECTED_NODES=$(kubectl get nodes --no-headers 2>/dev/null | grep "Ready" | wc -l)
+
+for NODE_ID in "$@"; do
+    echo "=== Replacing node $NODE_ID ==="
+    scw k8s node replace "$NODE_ID" || { echo "Replace failed for $NODE_ID"; exit 1; }
+
+    echo "  Waiting for old node to drain..."
+    sleep 30
+
+    wait_for_node "$EXPECTED_NODES"
+    wait_for_coredns
+
+    echo "  Verifying DNS..."
+    kubectl run dns-verify-$RANDOM --rm -it --restart=Never --image=busybox:1.36 \
+        -- sh -c "nslookup google.com > /dev/null 2>&1 && echo DNS_OK || echo DNS_FAIL" 2>/dev/null | grep -q "DNS_OK"
+
+    if [ $? -eq 0 ]; then
+        echo "  DNS verified OK"
+    else
+        echo "  WARNING: DNS verification failed — stopping. Fix manually before continuing."
+        exit 1
+    fi
+
+    echo "  Node $NODE_ID replaced successfully"
+    echo ""
+done
+
+echo "=== All nodes replaced safely ==="