From 5ca26d2da0c6461c130ef4731ac841fcbf2c4efe Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 13 Mar 2026 12:09:10 +0100 Subject: [PATCH] fix(infra): use runtime image for gpu-warmup (Argo emissary compat) busybox doesn't support Argo's emissary executor protocol, causing the wait sidecar to hang indefinitely and hold the GPU allocation. Switch to foxhunt-runtime which has proper /bin/sh and lets the sidecar detect container exit. Also adds nvidia-smi check during warmup. Co-Authored-By: Claude Opus 4.6 --- infra/k8s/argo/gpu-test-pipeline-template.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/infra/k8s/argo/gpu-test-pipeline-template.yaml b/infra/k8s/argo/gpu-test-pipeline-template.yaml index ac3497bee..caff57814 100644 --- a/infra/k8s/argo/gpu-test-pipeline-template.yaml +++ b/infra/k8s/argo/gpu-test-pipeline-template.yaml @@ -61,6 +61,8 @@ spec: dependencies: [gpu-warmup] # ── gpu-warmup: trigger H100 autoscale ── + # Requests GPU to force autoscaler to provision node, then releases it. + # Uses the runtime image (not busybox) for Argo emissary executor compat. - name: gpu-warmup nodeSelector: k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" @@ -72,12 +74,13 @@ spec: operator: Exists effect: NoSchedule container: - image: busybox:1.37 + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest command: ["/bin/sh", "-c"] args: - | echo "GPU warmup: triggering node autoscale..." - echo "GPU node scheduled, exiting to free resources" + nvidia-smi --query-gpu=name,memory.total --format=csv,noheader || true + echo "GPU node ready, releasing for compile-and-test" resources: requests: nvidia.com/gpu: "1"