From 5678d8ec2c270c720c107341d4e9bf313c0130cc Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 27 Feb 2026 19:57:07 +0100 Subject: [PATCH] fix(ci): move nvidia runtime to per-job override for deploy compatibility Deploy job was failing with "no runtime for nvidia" because the runner set runtime_class_name=nvidia globally, but deploy routes to the services pool which has no GPU/nvidia runtime. Fix: removed global runtime_class_name from runner config, added runtime_class_name_overwrite_allowed=".*", and set KUBERNETES_RUNTIME_CLASS_NAME=nvidia in GPU job templates (.rust-base, .train-rl-base, .train-validate-base). Co-Authored-By: Claude Opus 4.6 --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6aeb2684b..3f51b15ea 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -155,6 +155,8 @@ build-infra-runner: - kapsule - rust variables: + # GPU: compile jobs need CUDA for ml crate linking + KUBERNETES_RUNTIME_CLASS_NAME: nvidia # Give compile pods all available CPU for fast Rust compilation KUBERNETES_CPU_REQUEST: "7000m" KUBERNETES_CPU_LIMIT: "7800m" @@ -423,6 +425,7 @@ build-training: # Picked up by gitlab-runner-rl which mounts training-data-l4-pvc + sccache-l4-pvc # Runner default node_selector = ci-compile (L4-1-24G) variables: + KUBERNETES_RUNTIME_CLASS_NAME: nvidia # RL is single-threaded (~1 core actual), 2000m allows 2 concurrent on L4 KUBERNETES_CPU_REQUEST: "2000m" KUBERNETES_CPU_LIMIT: "3800m" @@ -452,6 +455,7 @@ build-training: - kapsule - gpu variables: + KUBERNETES_RUNTIME_CLASS_NAME: nvidia # Route to L40S pool (48GB VRAM for large supervised models) KUBERNETES_NODE_SELECTOR_POOL: "k8s.scaleway.com/pool-name=ci-training" # Supervised is GPU-bound — minimal CPU. Low request allows 3-4 concurrent jobs.