From 73b368a0de42530b853f12ee508ea3aaa5d264aa Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Thu, 26 Feb 2026 13:04:52 +0100
Subject: [PATCH] fix(ci): detect GPU at runtime, partition sccache by compute
 cap

The CI runner pool has both L4 (sm_89) and H100 (sm_90) nodes.
Hardcoding CUDA_COMPUTE_CAP causes PTX mismatches when sccache
serves objects compiled for the wrong architecture.

- Detect compute capability via nvidia-smi at job start
- Partition sccache into /mnt/sccache/sm_89 and sm_90
- Each GPU type maintains its own warm cache

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .gitlab-ci.yml | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 87f073067..981020287 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,13 +14,13 @@ stages:
 variables:
   SQLX_OFFLINE: "true"
   CARGO_TERM_COLOR: always
-  # H100 = compute capability 9.0 — tells bindgen_cuda to skip nvidia-smi detection
-  CUDA_COMPUTE_CAP: "90"
+  # CUDA_COMPUTE_CAP is set at runtime by detect_gpu() — L4=89, H100=90
+  # Do NOT hardcode here; sccache is partitioned by compute cap below
   # Stable JWT secret for api_gateway tests (prevents env-var race between parallel tests)
   JWT_SECRET: "CiTestSecret_Kx7mP9nR2sW5vY8bC3fG6jH1kL4pQ7tZ0uN9dM5eV8xS2wT6yA4zB_64chars_min"
   # Training data on block storage PVC (mounted at /mnt/training-data by runner)
   FOXHUNT_DATA_DIR: "/mnt/training-data/futures-baseline"
-  # sccache on local block storage PVC (mounted at /mnt/sccache by runner)
+  # sccache base dir on PVC — GPU stages override to /mnt/sccache/sm_<cap>
   SCCACHE_DIR: "/mnt/sccache"
   RUSTC_WRAPPER: /usr/local/bin/sccache
   # S3 credentials for Kaniko registry auth and Dockerfile builds
@@ -211,6 +211,11 @@ test:
     # Remove CUDA stubs from LD_LIBRARY_PATH — nvidia RuntimeClass provides real libcuda.so
     - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
     - nvidia-smi  # verify GPU access
+    # Detect GPU compute capability and partition sccache per architecture
+    - export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.')
+    - echo "Detected CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
+    - export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}"
+    - mkdir -p "$SCCACHE_DIR"
     # Wait for Redis sidecar (K8s executor doesn't wait for service readiness)
     - |
       for i in $(seq 1 30); do
@@ -219,8 +224,6 @@ test:
         fi
         echo "Waiting for Redis... ($i/30)"; sleep 1
       done
-    # Purge sccache — stale PTX from L4 (sm_89) causes CUDA_ERROR_INVALID_PTX on H100 (sm_90)
-    - rm -rf "${SCCACHE_DIR:?}"/* || true
     - sccache --zero-stats || true
     - cargo test --workspace --lib -- --skip model_loader::tests
     - sccache --show-stats || true
@@ -247,6 +250,11 @@ compile-services:
         - infra/docker/Dockerfile.training-runtime
         - .gitlab-ci.yml
   script:
+    # Detect GPU compute capability and partition sccache per architecture
+    - export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.')
+    - echo "Detected CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
+    - export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}"
+    - mkdir -p "$SCCACHE_DIR"
     - sccache --zero-stats || true
     # 1) Build service binaries (no CUDA feature)
     - cargo build --release