From 4e0dc97041153054f825088e4aa495f445574ac4 Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Fri, 27 Feb 2026 20:42:25 +0100
Subject: [PATCH] fix(ci): decouple compile from nvidia runtime

Compile only needs CUDA stubs (baked into ci-builder) + CUDA_COMPUTE_CAP
for candle-kernels PTX generation. No nvidia-smi or real GPU needed.

- Hardcode CUDA_COMPUTE_CAP=89 (L4) in .rust-base variables
- Remove KUBERNETES_RUNTIME_CLASS_NAME from .rust-base
- compile-services: skip LD_LIBRARY_PATH stripping and nvidia-smi
- test: graceful fallback when nvidia-smi unavailable

This unblocks compile on nodes without nvidia RuntimeClass handler
and opens the door to using CPU-only nodes for faster compilation.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .gitlab-ci.yml | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index dc10ea662..615bb6134 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -159,8 +159,9 @@ build-infra-runner:
     - kapsule
     - rust
   variables:
-    # GPU: compile jobs need CUDA for ml crate linking
-    KUBERNETES_RUNTIME_CLASS_NAME: nvidia
+    # Compile uses CUDA stubs from ci-builder image — no nvidia runtime needed.
+    # L4 GPU = compute capability 8.9 → candle-kernels generates sm_89 PTX
+    CUDA_COMPUTE_CAP: "89"
     # Give compile pods all available CPU for fast Rust compilation
     KUBERNETES_CPU_REQUEST: "7000m"
     KUBERNETES_CPU_LIMIT: "7800m"
@@ -200,11 +201,15 @@ test:
     - name: redis:7-alpine
       alias: redis
   script:
-    # All CI pods have runtime_class_name=nvidia → real GPU available
-    # Strip CUDA stubs so we use the real libcuda.so from nvidia runtime
-    - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
-    - export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.')
-    - echo "GPU=$(nvidia-smi --query-gpu=name --format=csv,noheader) CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
+    # Detect GPU if available (nvidia runtime), otherwise use CUDA_COMPUTE_CAP from variables
+    - |
+      if command -v nvidia-smi &>/dev/null; then
+        export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
+        export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.')
+        echo "GPU=$(nvidia-smi --query-gpu=name --format=csv,noheader) CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
+      else
+        echo "No nvidia-smi — using CUDA stubs, CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
+      fi
     - export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}"
     - mkdir -p "$SCCACHE_DIR"
     # Wait for Redis sidecar (K8s executor doesn't wait for service readiness)
@@ -238,10 +243,8 @@ compile-services:
         - infra/docker/Dockerfile.*
     - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
   script:
-    # All CI pods have runtime_class_name=nvidia → real GPU available
-    - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
-    - export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.')
-    - echo "GPU=$(nvidia-smi --query-gpu=name --format=csv,noheader) CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
+    # Compile uses CUDA stubs — no nvidia runtime needed, CUDA_COMPUTE_CAP from variables
+    - echo "CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
     - export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}"
     - mkdir -p "$SCCACHE_DIR"
     - sccache --zero-stats || true