foxhunt/infra/k8s/dns/node-local-dns.yaml

# Node Bootstrap DaemonSet for Kapsule
#
# Two problems on Kapsule nodes:
# 1. Nodes have nameserver 127.0.53.53 which can't resolve .svc.cluster.local
# 2. Containerd defaults to HTTPS for non-localhost registries
#
# Fixes:
# 1. Point /etc/resolv.conf to kube-dns ClusterIP (10.32.0.10)
# 2. Create /etc/containerd/certs.d/ config for HTTP GitLab registry
# 3. On GPU nodes (containerd v3 with conf.d/), add registry drop-in
#
# Init container sets up containerd registry config (one-time).
# Main container manages resolv.conf (watches for Kapsule resets).
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: node-bootstrap
  namespace: kube-system
  labels:
    app.kubernetes.io/name: node-bootstrap
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: node-bootstrap
  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
  template:
    metadata:
      labels:
        app.kubernetes.io/name: node-bootstrap
    spec:
      hostNetwork: true
      dnsPolicy: Default
      priorityClassName: system-node-critical
      tolerations:
        - operator: Exists
      hostPID: true
      initContainers:
        # Configure containerd to use HTTP for the GitLab registry.
        # On GPU nodes (NVIDIA overlay), also creates a v3-compatible conf.d drop-in
        # and restarts containerd so it picks up the new config.
        - name: setup-registry
          image: alpine:3.19
          securityContext:
            privileged: true
          command: ["/bin/sh", "-c"]
          args:
            - |
              set -e
              NEEDS_RESTART=false

              # 1. Create per-host containerd registry config (HTTP for GitLab)
              REGISTRY_DIR="/host-etc/containerd/certs.d/gitlab-registry.foxhunt.svc.cluster.local:5000"
              mkdir -p "$REGISTRY_DIR"
              DESIRED='server = "http://gitlab-registry.foxhunt.svc.cluster.local:5000"

              [host."http://gitlab-registry.foxhunt.svc.cluster.local:5000"]
                capabilities = ["pull", "resolve"]'
              DESIRED=$(printf '%s' "$DESIRED" | sed 's/^              //')
              if [ ! -f "$REGISTRY_DIR/hosts.toml" ] || [ "$(cat "$REGISTRY_DIR/hosts.toml")" != "$DESIRED" ]; then
                printf '%s\n' "$DESIRED" > "$REGISTRY_DIR/hosts.toml"
                echo "Created hosts.toml for HTTP gitlab-registry"
                NEEDS_RESTART=true
              else
                echo "hosts.toml already up to date"
              fi

              # 2. Ensure conf.d/ exists and always write the v3 registry drop-in.
              #    containerd 2.x may ignore the v2-style grpc.v1.cri.registry.config_path
              #    even when it's set in config.toml. The v1 images drop-in is the reliable path.
              #    Create conf.d/ if missing (base config already has imports = ["conf.d/*.toml"]).
              CONF_D="/host-etc/containerd/conf.d"
              mkdir -p "$CONF_D"
              if [ -d "$CONF_D" ]; then
                DROPIN='version = 3

              [plugins."io.containerd.cri.v1.images".registry]
                config_path = "/etc/containerd/certs.d"'
                DROPIN=$(printf '%s' "$DROPIN" | sed 's/^              //')
                if [ ! -f "$CONF_D/50-registry.toml" ] || [ "$(cat "$CONF_D/50-registry.toml")" != "$DROPIN" ]; then
                  printf '%s\n' "$DROPIN" > "$CONF_D/50-registry.toml"
                  echo "GPU node: created v3 registry config drop-in"
                  NEEDS_RESTART=true
                else
                  echo "GPU node: v3 drop-in already up to date"
                fi
              fi

              # 3. Restart containerd only if we changed config files.
              #    Uses /proc/1/root chroot to access host systemctl (requires hostPID + privileged).
              #    On a new node this will kill the DaemonSet pod; kubelet recreates it,
              #    second run finds files unchanged (NEEDS_RESTART=false), proceeds normally.
              if [ "$NEEDS_RESTART" = "true" ]; then
                echo "Restarting containerd to pick up config changes..."
                chroot /proc/1/root systemctl restart containerd || \
                  echo "WARNING: containerd restart failed (node may need manual restart)"
                echo "containerd restart issued"
              fi
          resources:
            requests:
              cpu: 10m
              memory: 8Mi
            limits:
              cpu: 50m
              memory: 16Mi
          volumeMounts:
            - name: host-etc
              mountPath: /host-etc
      containers:
        - name: resolv-manager
          image: busybox:1.37
          command: ["/bin/sh", "-c"]
          args:
            - |
              set -e
              RESOLV="/host-etc/resolv.conf"
              BACKUP="/host-etc/resolv.conf.pre-kube-dns"
              KUBE_DNS="10.32.0.10"

              restore() {
                if [ -f "$BACKUP" ]; then
                  cp "$BACKUP" "$RESOLV"
                  echo "$(date): Restored original resolv.conf"
                fi
                exit 0
              }
              trap restore TERM INT

              # Save original
              if [ ! -f "$BACKUP" ]; then
                cp "$RESOLV" "$BACKUP"
                echo "Saved original: $(cat "$BACKUP")"
              fi

              apply() {
                # CRITICAL: keep VPC upstream resolver as FALLBACK after kube-dns.
                # If resolv.conf ONLY has 10.32.0.10, node DNS is dead when CoreDNS
                # is down — containerd can't pull images, CoreDNS can't start,
                # entire cluster deadlocks (incident 2026-03-18, 2026-04-12).
                # Scaleway VPC gateway pushes 172.16.0.13 as DNS via DHCP.
                UPSTREAM=""
                if [ -f "$BACKUP" ]; then
                  UPSTREAM=$(grep "^nameserver" "$BACKUP" | grep -v "127.0.53.53\|$KUBE_DNS" | head -1 | awk '{print $2}')
                fi
                if [ -z "$UPSTREAM" ]; then
                  UPSTREAM="169.254.169.254"  # Scaleway metadata DNS fallback
                fi
                printf 'nameserver %s\nnameserver %s\nsearch foxhunt.svc.cluster.local svc.cluster.local cluster.local\noptions ndots:5 timeout:2 attempts:3\n' "$KUBE_DNS" "$UPSTREAM" > "$RESOLV"
                echo "$(date): resolv.conf → $KUBE_DNS + $UPSTREAM (fallback)"
              }

              # Initial apply
              apply

              # Watch for resets
              while true; do
                sleep 30 &
                wait $!
                if ! grep -q "$KUBE_DNS" "$RESOLV" 2>/dev/null; then
                  echo "$(date): resolv.conf was reset, re-applying"
                  apply
                fi
              done
          resources:
            requests:
              cpu: 5m
              memory: 8Mi
            limits:
              cpu: 10m
              memory: 16Mi
          volumeMounts:
            - name: host-etc
              mountPath: /host-etc
      terminationGracePeriodSeconds: 10
      volumes:
        - name: host-etc
          hostPath:
            path: /etc
            type: Directory