From 2bcfadf5f4d74ab1be5ec4113f91263d210ccbe4 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 26 May 2026 14:03:12 +0200 Subject: [PATCH] fix(infra): auto-detect CUDA_COMPUTE_CAP from nvidia-smi + show build stderr Sets CUDA_COMPUTE_CAP dynamically from the GPU on the node. Clears both ml-alpha and ml-core build caches. Captures build.rs stderr in pod logs for debugging. Co-Authored-By: Claude Opus 4.7 --- infra/k8s/argo/alpha-rl-template.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/infra/k8s/argo/alpha-rl-template.yaml b/infra/k8s/argo/alpha-rl-template.yaml index 18239efef..5a00bfcc4 100644 --- a/infra/k8s/argo/alpha-rl-template.yaml +++ b/infra/k8s/argo/alpha-rl-template.yaml @@ -156,12 +156,16 @@ spec: SHA=$(git rev-parse --short=9 HEAD) echo "=== Branch: $BRANCH SHA: $SHA ===" - # Clear stale build artifacts (cubin→fatbin migration) - rm -rf ${CARGO_TARGET_DIR}/build/ml-alpha-* + # Auto-detect GPU compute capability for all crates' build.rs + export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.') + echo "=== GPU arch: sm_${CUDA_COMPUTE_CAP} ===" + + # Clear stale build artifacts when arch changes + rm -rf ${CARGO_TARGET_DIR}/build/ml-alpha-* ${CARGO_TARGET_DIR}/build/ml-core-* # Compile (~90s cold, ~3s warm after first run) echo "=== Compile ===" - time cargo build --release -p ml-alpha --example alpha_rl_train + time cargo build --release -p ml-alpha --example alpha_rl_train 2>&1 # Train OUT="/feature-cache/alpha-rl-runs/$SHA/fold{{workflow.parameters.fold-idx}}"