gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ci): decouple compile from nvidia runtime

Browse Source 
Compile only needs CUDA stubs (baked into ci-builder) + CUDA_COMPUTE_CAP
for candle-kernels PTX generation. No nvidia-smi or real GPU needed.

- Hardcode CUDA_COMPUTE_CAP=89 (L4) in .rust-base variables
- Remove KUBERNETES_RUNTIME_CLASS_NAME from .rust-base
- compile-services: skip LD_LIBRARY_PATH stripping and nvidia-smi
- test: graceful fallback when nvidia-smi unavailable

This unblocks compile on nodes without nvidia RuntimeClass handler
and opens the door to using CPU-only nodes for faster compilation.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-27 20:42:25 +01:00
parent 312d113c47
commit 4e0dc97041
1 changed files with 14 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
.gitlab-ci.yml
View File

@@ -159,8 +159,9 @@ build-infra-runner:
- kapsule
- rust
variables:
# GPU: compile jobs need CUDA for ml crate linking
KUBERNETES_RUNTIME_CLASS_NAME: nvidia
# Compile uses CUDA stubs from ci-builder image — no nvidia runtime needed.
# L4 GPU = compute capability 8.9 → candle-kernels generates sm_89 PTX
CUDA_COMPUTE_CAP: "89"
# Give compile pods all available CPU for fast Rust compilation
KUBERNETES_CPU_REQUEST: "7000m"
KUBERNETES_CPU_LIMIT: "7800m"
@@ -200,11 +201,15 @@ test:
- name: redis:7-alpine
alias: redis
script:
# All CI pods have runtime_class_name=nvidia → real GPU available
# Strip CUDA stubs so we use the real libcuda.so from nvidia runtime
- export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
- export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.')
- echo "GPU=$(nvidia-smi --query-gpu=name --format=csv,noheader) CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
# Detect GPU if available (nvidia runtime), otherwise use CUDA_COMPUTE_CAP from variables
- |
if command -v nvidia-smi &>/dev/null; then
export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.')
echo "GPU=$(nvidia-smi --query-gpu=name --format=csv,noheader) CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
else
echo "No nvidia-smi — using CUDA stubs, CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
fi
- export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}"
- mkdir -p "$SCCACHE_DIR"
# Wait for Redis sidecar (K8s executor doesn't wait for service readiness)
@@ -238,10 +243,8 @@ compile-services:
- infra/docker/Dockerfile.*
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
script:
# All CI pods have runtime_class_name=nvidia → real GPU available
- export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
- export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.')
- echo "GPU=$(nvidia-smi --query-gpu=name --format=csv,noheader) CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
# Compile uses CUDA stubs — no nvidia runtime needed, CUDA_COMPUTE_CAP from variables
- echo "CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
- export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}"
- mkdir -p "$SCCACHE_DIR"
- sccache --zero-stats || true