diff --git a/infra/k8s/argo/training-workflow-template.yaml b/infra/k8s/argo/training-workflow-template.yaml index 55eb22c6a..9d3210e66 100644 --- a/infra/k8s/argo/training-workflow-template.yaml +++ b/infra/k8s/argo/training-workflow-template.yaml @@ -83,8 +83,8 @@ spec: template: upload-results dependencies: [evaluate] - # GPU warmup: triggers H100 autoscale during compilation so the node - # is ready when hyperopt starts. Exits immediately after nvidia-smi. + # GPU warmup: triggers GPU node autoscale during fetch-binary so the node + # and NVIDIA device plugin are ready when hyperopt starts. - name: gpu-warmup nodeSelector: k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" @@ -102,7 +102,7 @@ spec: args: - | echo "GPU warmup: triggering node autoscale..." - nvidia-smi --query-gpu=name,memory.total --format=csv,noheader + nvidia-smi --query-gpu=name,memory.total --format=csv,noheader 2>/dev/null || echo "nvidia-smi not yet available (driver still mounting)" echo "GPU node ready, exiting to free resources" resources: requests: