From 46d686ee6e1ced5c810d6970217f1c65d8f87be5 Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Fri, 27 Feb 2026 11:07:35 +0100
Subject: [PATCH] fix(ci): move nvidia RuntimeClass from runner default to
 per-job override

L4 ci-compile node was failing with 'no runtime for nvidia' because
fresh nodes take time to install nvidia drivers. Kaniko builds don't
need GPU at all. Now only Rust compile/test and training jobs request
nvidia RuntimeClass via KUBERNETES_RUNTIME_CLASS_NAME CI variable.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .gitlab-ci.yml                      | 3 +++
 infra/k8s/gitlab/runner-values.yaml | 6 ++++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 930cf9dfb..a030d3e6d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -160,6 +160,8 @@ build-infra-runner:
     KUBERNETES_CPU_LIMIT: "7800m"
     KUBERNETES_MEMORY_REQUEST: "16Gi"
     KUBERNETES_MEMORY_LIMIT: "80Gi"
+    # GPU runtime for CUDA compilation — Kaniko jobs don't set this
+    KUBERNETES_RUNTIME_CLASS_NAME: "nvidia"
   before_script:
     # Mold linker fallback — .cargo/config.toml uses -fuse-ld=mold via clang.
     # clang's -fuse-ld=mold searches for "ld.mold" (not "mold") in its linker paths.
@@ -444,6 +446,7 @@ build-training:
     KUBERNETES_CPU_LIMIT: "3500m"
     KUBERNETES_MEMORY_REQUEST: "16Gi"
     KUBERNETES_MEMORY_LIMIT: "40Gi"
+    KUBERNETES_RUNTIME_CLASS_NAME: "nvidia"
   rules:
     - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
       when: manual
diff --git a/infra/k8s/gitlab/runner-values.yaml b/infra/k8s/gitlab/runner-values.yaml
index d73d5e2f5..f408fd7a4 100644
--- a/infra/k8s/gitlab/runner-values.yaml
+++ b/infra/k8s/gitlab/runner-values.yaml
@@ -43,8 +43,10 @@ runners:
         helper_memory_request = "128Mi"
         helper_memory_limit = "512Mi"
         image_pull_secrets = ["scw-registry", "gitlab-registry"]
-        # Use nvidia RuntimeClass — injects GPU drivers + /dev/nvidia* via nvidia-container-runtime
-        runtime_class_name = "nvidia"
+        # GPU jobs set KUBERNETES_RUNTIME_CLASS_NAME=nvidia via CI variables
+        # Default empty — Kaniko builds don't need GPU runtime
+        runtime_class_name = ""
+        runtime_class_name_overwrite_allowed = ".*"
       # Sub-tables must come AFTER all scalar values (TOML rule)
       [runners.kubernetes.node_selector]
         "k8s.scaleway.com/pool-name" = "ci-compile"