diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dc10ea662..615bb6134 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -159,8 +159,9 @@ build-infra-runner: - kapsule - rust variables: - # GPU: compile jobs need CUDA for ml crate linking - KUBERNETES_RUNTIME_CLASS_NAME: nvidia + # Compile uses CUDA stubs from ci-builder image — no nvidia runtime needed. + # L4 GPU = compute capability 8.9 → candle-kernels generates sm_89 PTX + CUDA_COMPUTE_CAP: "89" # Give compile pods all available CPU for fast Rust compilation KUBERNETES_CPU_REQUEST: "7000m" KUBERNETES_CPU_LIMIT: "7800m" @@ -200,11 +201,15 @@ test: - name: redis:7-alpine alias: redis script: - # All CI pods have runtime_class_name=nvidia → real GPU available - # Strip CUDA stubs so we use the real libcuda.so from nvidia runtime - - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') - - export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.') - - echo "GPU=$(nvidia-smi --query-gpu=name --format=csv,noheader) CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP" + # Detect GPU if available (nvidia runtime), otherwise use CUDA_COMPUTE_CAP from variables + - | + if command -v nvidia-smi &>/dev/null; then + export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') + export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.') + echo "GPU=$(nvidia-smi --query-gpu=name --format=csv,noheader) CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP" + else + echo "No nvidia-smi — using CUDA stubs, CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP" + fi - export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}" - mkdir -p "$SCCACHE_DIR" # Wait for Redis sidecar (K8s executor doesn't wait for service readiness) @@ -238,10 +243,8 @@ compile-services: - infra/docker/Dockerfile.* - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api" script: - # All CI pods have runtime_class_name=nvidia → real GPU available - - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') - - export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.') - - echo "GPU=$(nvidia-smi --query-gpu=name --format=csv,noheader) CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP" + # Compile uses CUDA stubs — no nvidia runtime needed, CUDA_COMPUTE_CAP from variables + - echo "CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP" - export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}" - mkdir -p "$SCCACHE_DIR" - sccache --zero-stats || true