From 5f2d7d670aa2b37a42e3598b37d7223653df9bce Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 27 Feb 2026 12:02:28 +0100 Subject: [PATCH] fix(ci): make nvidia-smi optional for compile/test jobs Compile and test jobs run on ci-compile (L4) without nvidia RuntimeClass since they only need CUDA headers, not GPU access. The nvidia-smi call was failing because the GPU device isn't passed through without the RuntimeClass. Now nvidia-smi failure is non-fatal for compile/test, with CUDA compute cap defaulting to 89 (L4) for sccache partitioning. Training jobs still require nvidia-smi to succeed (they need actual GPU access). Co-Authored-By: Claude Opus 4.6 --- .gitlab-ci.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5d9eb5702..2223f5712 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -208,10 +208,12 @@ test: script: # Remove CUDA stubs from LD_LIBRARY_PATH — nvidia RuntimeClass provides real libcuda.so - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') - - nvidia-smi # verify GPU access + - nvidia-smi || echo "nvidia-smi not available (no GPU RuntimeClass) — compile-only mode" # Detect GPU compute capability and partition sccache per architecture - - export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.') - - echo "Detected CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP" + # Falls back to 89 (L4) when nvidia-smi unavailable (compile jobs don't need GPU) + - export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d '.' || echo "89") + - export CUDA_COMPUTE_CAP=${CUDA_COMPUTE_CAP:-89} + - echo "CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP" - export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}" - mkdir -p "$SCCACHE_DIR" # Wait for Redis sidecar (K8s executor doesn't wait for service readiness) @@ -249,8 +251,10 @@ compile-services: - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api" script: # Detect GPU compute capability and partition sccache per architecture - - export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.') - - echo "Detected CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP" + # Falls back to 89 (L4) when nvidia-smi unavailable (compile doesn't need GPU) + - export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d '.' || echo "89") + - export CUDA_COMPUTE_CAP=${CUDA_COMPUTE_CAP:-89} + - echo "CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP" - export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}" - mkdir -p "$SCCACHE_DIR" - sccache --zero-stats || true