gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ci): make nvidia-smi optional for compile/test jobs

Browse Source 
Compile and test jobs run on ci-compile (L4) without nvidia RuntimeClass
since they only need CUDA headers, not GPU access. The nvidia-smi call
was failing because the GPU device isn't passed through without the
RuntimeClass.

Now nvidia-smi failure is non-fatal for compile/test, with CUDA compute
cap defaulting to 89 (L4) for sccache partitioning. Training jobs still
require nvidia-smi to succeed (they need actual GPU access).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-27 12:02:28 +01:00
parent e51ec73201
commit 5f2d7d670a
1 changed files with 9 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
.gitlab-ci.yml
View File

@@ -208,10 +208,12 @@ test:
script:
# Remove CUDA stubs from LD_LIBRARY_PATH — nvidia RuntimeClass provides real libcuda.so
- export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
- nvidia-smi # verify GPU access
- nvidia-smi || echo "nvidia-smi not available (no GPU RuntimeClass) — compile-only mode"
# Detect GPU compute capability and partition sccache per architecture
- export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.')
- echo "Detected CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
# Falls back to 89 (L4) when nvidia-smi unavailable (compile jobs don't need GPU)
- export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d '.' || echo "89")
- export CUDA_COMPUTE_CAP=${CUDA_COMPUTE_CAP:-89}
- echo "CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
- export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}"
- mkdir -p "$SCCACHE_DIR"
# Wait for Redis sidecar (K8s executor doesn't wait for service readiness)
@@ -249,8 +251,10 @@ compile-services:
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
script:
# Detect GPU compute capability and partition sccache per architecture
- export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.')
- echo "Detected CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
# Falls back to 89 (L4) when nvidia-smi unavailable (compile doesn't need GPU)
- export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d '.' || echo "89")
- export CUDA_COMPUTE_CAP=${CUDA_COMPUTE_CAP:-89}
- echo "CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
- export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}"
- mkdir -p "$SCCACHE_DIR"
- sccache --zero-stats || true