diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 87f073067..981020287 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,13 +14,13 @@ stages: variables: SQLX_OFFLINE: "true" CARGO_TERM_COLOR: always - # H100 = compute capability 9.0 — tells bindgen_cuda to skip nvidia-smi detection - CUDA_COMPUTE_CAP: "90" + # CUDA_COMPUTE_CAP is set at runtime by detect_gpu() — L4=89, H100=90 + # Do NOT hardcode here; sccache is partitioned by compute cap below # Stable JWT secret for api_gateway tests (prevents env-var race between parallel tests) JWT_SECRET: "CiTestSecret_Kx7mP9nR2sW5vY8bC3fG6jH1kL4pQ7tZ0uN9dM5eV8xS2wT6yA4zB_64chars_min" # Training data on block storage PVC (mounted at /mnt/training-data by runner) FOXHUNT_DATA_DIR: "/mnt/training-data/futures-baseline" - # sccache on local block storage PVC (mounted at /mnt/sccache by runner) + # sccache base dir on PVC — GPU stages override to /mnt/sccache/sm_ SCCACHE_DIR: "/mnt/sccache" RUSTC_WRAPPER: /usr/local/bin/sccache # S3 credentials for Kaniko registry auth and Dockerfile builds @@ -211,6 +211,11 @@ test: # Remove CUDA stubs from LD_LIBRARY_PATH — nvidia RuntimeClass provides real libcuda.so - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') - nvidia-smi # verify GPU access + # Detect GPU compute capability and partition sccache per architecture + - export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.') + - echo "Detected CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP" + - export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}" + - mkdir -p "$SCCACHE_DIR" # Wait for Redis sidecar (K8s executor doesn't wait for service readiness) - | for i in $(seq 1 30); do @@ -219,8 +224,6 @@ test: fi echo "Waiting for Redis... ($i/30)"; sleep 1 done - # Purge sccache — stale PTX from L4 (sm_89) causes CUDA_ERROR_INVALID_PTX on H100 (sm_90) - - rm -rf "${SCCACHE_DIR:?}"/* || true - sccache --zero-stats || true - cargo test --workspace --lib -- --skip model_loader::tests - sccache --show-stats || true @@ -247,6 +250,11 @@ compile-services: - infra/docker/Dockerfile.training-runtime - .gitlab-ci.yml script: + # Detect GPU compute capability and partition sccache per architecture + - export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.') + - echo "Detected CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP" + - export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}" + - mkdir -p "$SCCACHE_DIR" - sccache --zero-stats || true # 1) Build service binaries (no CUDA feature) - cargo build --release