diff --git a/infra/k8s/argo/training-workflow-template.yaml b/infra/k8s/argo/training-workflow-template.yaml index 9e12d6bd2..c9788daba 100644 --- a/infra/k8s/argo/training-workflow-template.yaml +++ b/infra/k8s/argo/training-workflow-template.yaml @@ -66,9 +66,11 @@ spec: tasks: - name: fetch-binary template: fetch-binary + - name: gpu-warmup + template: gpu-warmup - name: hyperopt template: hyperopt - dependencies: [fetch-binary] + dependencies: [fetch-binary, gpu-warmup] - name: train-best template: train-best dependencies: [hyperopt] @@ -79,6 +81,36 @@ spec: template: upload-results dependencies: [evaluate] + # GPU warmup: triggers H100 autoscale during compilation so the node + # is ready when hyperopt starts. Exits immediately after nvidia-smi. + - name: gpu-warmup + nodeSelector: + k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + - key: node.cilium.io/agent-not-ready + operator: Exists + effect: NoSchedule + container: + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest + command: ["/bin/sh", "-c"] + args: + - | + echo "GPU warmup: triggering node autoscale..." + nvidia-smi --query-gpu=name,memory.total --format=csv,noheader + echo "GPU node ready, exiting to free resources" + resources: + requests: + nvidia.com/gpu: "1" + cpu: 100m + memory: 128Mi + limits: + nvidia.com/gpu: "1" + cpu: 200m + memory: 256Mi + - name: fetch-binary nodeSelector: k8s.scaleway.com/pool-name: platform