From 5d5cce72d2d7bbd10f5263db1fe95b1f99087508 Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Thu, 5 Mar 2026 22:12:17 +0100
Subject: [PATCH] fix(infra): auto-restart containerd on GPU nodes for registry
 config

DaemonSet init container now:
- Detects GPU nodes (99-nvidia.toml in conf.d)
- Creates v3-compatible registry config drop-in
- Restarts containerd via chroot if config files changed
- Idempotent: skips restart if files already exist

Requires hostPID + privileged for chroot /proc/1/root.
On new nodes, first run creates files + restarts containerd
(kills pod), second run finds files unchanged (no restart).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 infra/k8s/dns/node-local-dns.yaml | 62 ++++++++++++++++++++++---------
 1 file changed, 44 insertions(+), 18 deletions(-)

diff --git a/infra/k8s/dns/node-local-dns.yaml b/infra/k8s/dns/node-local-dns.yaml
index 3ba62fc62..29184c00f 100644
--- a/infra/k8s/dns/node-local-dns.yaml
+++ b/infra/k8s/dns/node-local-dns.yaml
@@ -36,39 +36,65 @@ spec:
       priorityClassName: system-node-critical
       tolerations:
         - operator: Exists
+      hostPID: true
       initContainers:
-        # Configure containerd to use HTTP for the GitLab registry
+        # Configure containerd to use HTTP for the GitLab registry.
+        # On GPU nodes (NVIDIA overlay), also creates a v3-compatible conf.d drop-in
+        # and restarts containerd so it picks up the new config.
         - name: setup-registry
-          image: busybox:1.36
+          image: alpine:3.19
+          securityContext:
+            privileged: true
           command: ["/bin/sh", "-c"]
           args:
             - |
               set -e
+              NEEDS_RESTART=false
+
+              # 1. Create per-host containerd registry config (HTTP for GitLab)
               REGISTRY_DIR="/host-etc/containerd/certs.d/gitlab-registry.foxhunt.svc.cluster.local:5000"
               mkdir -p "$REGISTRY_DIR"
-              cat > "$REGISTRY_DIR/hosts.toml" << 'EOF'
-              server = "http://gitlab-registry.foxhunt.svc.cluster.local:5000"
+              DESIRED='server = "http://gitlab-registry.foxhunt.svc.cluster.local:5000"
 
               [host."http://gitlab-registry.foxhunt.svc.cluster.local:5000"]
-                capabilities = ["pull", "resolve"]
-              EOF
-              sed -i 's/^              //' "$REGISTRY_DIR/hosts.toml"
-              echo "Configured containerd registry: HTTP for gitlab-registry"
-              cat "$REGISTRY_DIR/hosts.toml"
+                capabilities = ["pull", "resolve"]'
+              DESIRED=$(printf '%s' "$DESIRED" | sed 's/^              //')
+              if [ ! -f "$REGISTRY_DIR/hosts.toml" ] || [ "$(cat "$REGISTRY_DIR/hosts.toml")" != "$DESIRED" ]; then
+                printf '%s\n' "$DESIRED" > "$REGISTRY_DIR/hosts.toml"
+                echo "Created hosts.toml for HTTP gitlab-registry"
+                NEEDS_RESTART=true
+              else
+                echo "hosts.toml already up to date"
+              fi
 
-              # GPU nodes (NVIDIA overlay) use containerd v2 with version=3 config imports.
-              # The old grpc.v1.cri.registry path is ignored — must use cri.v1.images.
+              # 2. GPU nodes: containerd v2 with NVIDIA overlay uses version=3 config naming.
+              #    The old grpc.v1.cri.registry.config_path is silently ignored.
+              #    Must add cri.v1.images.registry drop-in.
               CONF_D="/host-etc/containerd/conf.d"
               if [ -d "$CONF_D" ] && [ -f "$CONF_D/99-nvidia.toml" ]; then
-                cat > "$CONF_D/50-registry.toml" << 'REGEOF'
-              version = 3
+                DROPIN='version = 3
 
               [plugins."io.containerd.cri.v1.images".registry]
-                config_path = "/etc/containerd/certs.d"
-              REGEOF
-                sed -i 's/^              //' "$CONF_D/50-registry.toml"
-                echo "GPU node detected: added v3-compatible registry config"
-                cat "$CONF_D/50-registry.toml"
+                config_path = "/etc/containerd/certs.d"'
+                DROPIN=$(printf '%s' "$DROPIN" | sed 's/^              //')
+                if [ ! -f "$CONF_D/50-registry.toml" ] || [ "$(cat "$CONF_D/50-registry.toml")" != "$DROPIN" ]; then
+                  printf '%s\n' "$DROPIN" > "$CONF_D/50-registry.toml"
+                  echo "GPU node: created v3 registry config drop-in"
+                  NEEDS_RESTART=true
+                else
+                  echo "GPU node: v3 drop-in already up to date"
+                fi
+              fi
+
+              # 3. Restart containerd only if we changed config files.
+              #    Uses /proc/1/root chroot to access host systemctl (requires hostPID + privileged).
+              #    On a new node this will kill the DaemonSet pod; kubelet recreates it,
+              #    second run finds files unchanged (NEEDS_RESTART=false), proceeds normally.
+              if [ "$NEEDS_RESTART" = "true" ]; then
+                echo "Restarting containerd to pick up config changes..."
+                chroot /proc/1/root systemctl restart containerd || \
+                  echo "WARNING: containerd restart failed (node may need manual restart)"
+                echo "containerd restart issued"
               fi
           resources:
             requests: