feat(infra): add GPU warmup step to training pipeline

Runs nvidia-smi on GPU node in parallel with fetch-binary, triggering
H100 autoscale during compilation so the node is ready when hyperopt
starts. Exits immediately to free GPU resources.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

infra/k8s/argo/training-workflow-template.yaml

@@ -66,9 +66,11 @@ spec:
         tasks:
           - name: fetch-binary
             template: fetch-binary
+          - name: gpu-warmup
+            template: gpu-warmup
           - name: hyperopt
             template: hyperopt
-            dependencies: [fetch-binary]
+            dependencies: [fetch-binary, gpu-warmup]
           - name: train-best
             template: train-best
             dependencies: [hyperopt]
@@ -79,6 +81,36 @@ spec:
             template: upload-results
             dependencies: [evaluate]
 
+    # GPU warmup: triggers H100 autoscale during compilation so the node
+    # is ready when hyperopt starts. Exits immediately after nvidia-smi.
+    - name: gpu-warmup
+      nodeSelector:
+        k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
+      tolerations:
+        - key: nvidia.com/gpu
+          operator: Exists
+          effect: NoSchedule
+        - key: node.cilium.io/agent-not-ready
+          operator: Exists
+          effect: NoSchedule
+      container:
+        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
+        command: ["/bin/sh", "-c"]
+        args:
+          - |
+            echo "GPU warmup: triggering node autoscale..."
+            nvidia-smi --query-gpu=name,memory.total --format=csv,noheader
+            echo "GPU node ready, exiting to free resources"
+      resources:
+        requests:
+          nvidia.com/gpu: "1"
+          cpu: 100m
+          memory: 128Mi
+        limits:
+          nvidia.com/gpu: "1"
+          cpu: 200m
+          memory: 256Mi
+
     - name: fetch-binary
       nodeSelector:
         k8s.scaleway.com/pool-name: platform