fix(infra): use runtime image for gpu-warmup (Argo emissary compat)
busybox doesn't support Argo's emissary executor protocol, causing the wait sidecar to hang indefinitely and hold the GPU allocation. Switch to foxhunt-runtime which has proper /bin/sh and lets the sidecar detect container exit. Also adds nvidia-smi check during warmup. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -61,6 +61,8 @@ spec:
|
||||
dependencies: [gpu-warmup]
|
||||
|
||||
# ── gpu-warmup: trigger H100 autoscale ──
|
||||
# Requests GPU to force autoscaler to provision node, then releases it.
|
||||
# Uses the runtime image (not busybox) for Argo emissary executor compat.
|
||||
- name: gpu-warmup
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
|
||||
@@ -72,12 +74,13 @@ spec:
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
container:
|
||||
image: busybox:1.37
|
||||
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
echo "GPU warmup: triggering node autoscale..."
|
||||
echo "GPU node scheduled, exiting to free resources"
|
||||
nvidia-smi --query-gpu=name,memory.total --format=csv,noheader || true
|
||||
echo "GPU node ready, releasing for compile-and-test"
|
||||
resources:
|
||||
requests:
|
||||
nvidia.com/gpu: "1"
|
||||
|
||||
Reference in New Issue
Block a user