From fbb674f9e2f3c797e6d749c4c637925d4da7938e Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 7 Mar 2026 00:56:35 +0100 Subject: [PATCH] fix(infra): make gpu-warmup tolerant of missing nvidia-smi nvidia-smi is driver-mounted by the GPU operator, which may not be ready when the warmup pod starts on a fresh autoscaled node. The warmup's purpose is just triggering autoscale, not GPU validation. Co-Authored-By: Claude Opus 4.6 --- infra/k8s/argo/training-workflow-template.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/k8s/argo/training-workflow-template.yaml b/infra/k8s/argo/training-workflow-template.yaml index 55eb22c6a..9d3210e66 100644 --- a/infra/k8s/argo/training-workflow-template.yaml +++ b/infra/k8s/argo/training-workflow-template.yaml @@ -83,8 +83,8 @@ spec: template: upload-results dependencies: [evaluate] - # GPU warmup: triggers H100 autoscale during compilation so the node - # is ready when hyperopt starts. Exits immediately after nvidia-smi. + # GPU warmup: triggers GPU node autoscale during fetch-binary so the node + # and NVIDIA device plugin are ready when hyperopt starts. - name: gpu-warmup nodeSelector: k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" @@ -102,7 +102,7 @@ spec: args: - | echo "GPU warmup: triggering node autoscale..." - nvidia-smi --query-gpu=name,memory.total --format=csv,noheader + nvidia-smi --query-gpu=name,memory.total --format=csv,noheader 2>/dev/null || echo "nvidia-smi not yet available (driver still mounting)" echo "GPU node ready, exiting to free resources" resources: requests: